2024-12-03 09:13:39,042 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 09:13:39,054 main DEBUG Took 0.010838 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 09:13:39,055 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 09:13:39,055 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 09:13:39,056 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 09:13:39,057 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:13:39,066 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 09:13:39,079 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,081 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:13:39,082 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,082 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:13:39,083 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:13:39,084 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,085 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:13:39,085 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:13:39,087 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:13:39,088 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:13:39,089 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:13:39,090 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:13:39,091 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:13:39,092 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:13:39,093 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,093 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:13:39,094 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,094 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 09:13:39,096 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:13:39,097 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 09:13:39,100 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 09:13:39,100 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 09:13:39,102 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 09:13:39,102 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 09:13:39,111 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 09:13:39,113 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 09:13:39,115 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 09:13:39,115 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 09:13:39,115 main DEBUG createAppenders(={Console}) 2024-12-03 09:13:39,116 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-03 09:13:39,116 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 09:13:39,117 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-03 09:13:39,117 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 09:13:39,117 main DEBUG OutputStream closed 2024-12-03 09:13:39,118 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 09:13:39,118 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 09:13:39,118 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-03 09:13:39,198 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 09:13:39,201 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 09:13:39,202 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 09:13:39,203 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 09:13:39,204 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 09:13:39,205 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 09:13:39,205 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 09:13:39,205 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 09:13:39,206 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 09:13:39,206 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 09:13:39,207 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 09:13:39,207 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 09:13:39,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 09:13:39,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 09:13:39,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 09:13:39,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 09:13:39,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 09:13:39,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 09:13:39,213 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 09:13:39,214 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-03 09:13:39,214 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 09:13:39,215 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-03T09:13:39,459 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3 2024-12-03 09:13:39,462 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 09:13:39,462 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T09:13:39,469 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-03T09:13:39,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T09:13:39,500 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/cluster_265ee787-60f6-c6d8-cc9e-dd886dec81cb, deleteOnExit=true 2024-12-03T09:13:39,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T09:13:39,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/test.cache.data in system properties and HBase conf 2024-12-03T09:13:39,502 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T09:13:39,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/hadoop.log.dir in system properties and HBase conf 2024-12-03T09:13:39,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T09:13:39,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T09:13:39,504 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T09:13:39,584 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T09:13:39,665 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T09:13:39,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T09:13:39,669 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T09:13:39,669 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T09:13:39,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T09:13:39,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T09:13:39,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T09:13:39,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T09:13:39,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T09:13:39,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T09:13:39,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/nfs.dump.dir in system properties and HBase conf 2024-12-03T09:13:39,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/java.io.tmpdir in system properties and HBase conf 2024-12-03T09:13:39,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T09:13:39,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T09:13:39,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T09:13:40,674 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T09:13:40,742 INFO [Time-limited test {}] log.Log(170): Logging initialized @2355ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T09:13:40,804 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T09:13:40,851 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T09:13:40,868 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T09:13:40,868 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T09:13:40,869 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T09:13:40,880 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T09:13:40,882 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b490999{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/hadoop.log.dir/,AVAILABLE} 2024-12-03T09:13:40,882 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@707ebc91{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T09:13:41,036 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5f062aca{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/java.io.tmpdir/jetty-localhost-36865-hadoop-hdfs-3_4_1-tests_jar-_-any-11198958594654315123/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T09:13:41,043 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4fee885d{HTTP/1.1, (http/1.1)}{localhost:36865} 2024-12-03T09:13:41,043 INFO [Time-limited test {}] server.Server(415): Started @2657ms 2024-12-03T09:13:41,547 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T09:13:41,554 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T09:13:41,556 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T09:13:41,556 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T09:13:41,556 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T09:13:41,557 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@849d63d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/hadoop.log.dir/,AVAILABLE} 2024-12-03T09:13:41,557 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a247cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T09:13:41,652 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@53343f2d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/java.io.tmpdir/jetty-localhost-45987-hadoop-hdfs-3_4_1-tests_jar-_-any-7038983173401900677/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T09:13:41,653 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3ff8ab72{HTTP/1.1, (http/1.1)}{localhost:45987} 2024-12-03T09:13:41,654 INFO [Time-limited test {}] server.Server(415): Started @3267ms 2024-12-03T09:13:41,699 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T09:13:42,648 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/cluster_265ee787-60f6-c6d8-cc9e-dd886dec81cb/data/data2/current/BP-777680306-172.17.0.2-1733217220182/current, will proceed with Du for space computation calculation, 2024-12-03T09:13:42,648 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/cluster_265ee787-60f6-c6d8-cc9e-dd886dec81cb/data/data1/current/BP-777680306-172.17.0.2-1733217220182/current, will proceed with Du for space computation calculation, 2024-12-03T09:13:42,676 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T09:13:42,718 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1b4f7dda1cda59f1 with lease ID 0xc4b6b3e346d6ffe3: Processing first storage report for DS-54ab912e-b045-4c7b-b806-dfaa9a0f3e78 from datanode DatanodeRegistration(127.0.0.1:39265, datanodeUuid=df211a26-18bf-4a7e-a99b-44cda7c05900, infoPort=36147, infoSecurePort=0, ipcPort=33415, storageInfo=lv=-57;cid=testClusterID;nsid=830123464;c=1733217220182) 2024-12-03T09:13:42,720 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1b4f7dda1cda59f1 with lease ID 0xc4b6b3e346d6ffe3: from storage DS-54ab912e-b045-4c7b-b806-dfaa9a0f3e78 node DatanodeRegistration(127.0.0.1:39265, datanodeUuid=df211a26-18bf-4a7e-a99b-44cda7c05900, infoPort=36147, infoSecurePort=0, ipcPort=33415, storageInfo=lv=-57;cid=testClusterID;nsid=830123464;c=1733217220182), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-12-03T09:13:42,721 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1b4f7dda1cda59f1 with lease ID 0xc4b6b3e346d6ffe3: Processing first storage report for DS-3c6adb36-1db5-4f23-8da5-a5b5f6d669f7 from datanode DatanodeRegistration(127.0.0.1:39265, datanodeUuid=df211a26-18bf-4a7e-a99b-44cda7c05900, infoPort=36147, infoSecurePort=0, ipcPort=33415, storageInfo=lv=-57;cid=testClusterID;nsid=830123464;c=1733217220182) 2024-12-03T09:13:42,721 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1b4f7dda1cda59f1 with lease ID 0xc4b6b3e346d6ffe3: from storage DS-3c6adb36-1db5-4f23-8da5-a5b5f6d669f7 node DatanodeRegistration(127.0.0.1:39265, datanodeUuid=df211a26-18bf-4a7e-a99b-44cda7c05900, infoPort=36147, infoSecurePort=0, ipcPort=33415, storageInfo=lv=-57;cid=testClusterID;nsid=830123464;c=1733217220182), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T09:13:42,801 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3 2024-12-03T09:13:42,944 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/cluster_265ee787-60f6-c6d8-cc9e-dd886dec81cb/zookeeper_0, clientPort=58590, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/cluster_265ee787-60f6-c6d8-cc9e-dd886dec81cb/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/cluster_265ee787-60f6-c6d8-cc9e-dd886dec81cb/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T09:13:42,952 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58590 2024-12-03T09:13:42,960 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:13:42,962 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:13:43,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741825_1001 (size=7) 2024-12-03T09:13:43,561 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09 with version=8 2024-12-03T09:13:43,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/hbase-staging 2024-12-03T09:13:43,643 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T09:13:43,860 INFO [Time-limited test {}] client.ConnectionUtils(128): master/699c9baf3d29:0 server-side Connection retries=45 2024-12-03T09:13:43,869 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T09:13:43,869 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T09:13:43,873 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T09:13:43,873 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T09:13:43,874 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T09:13:44,003 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T09:13:44,056 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T09:13:44,064 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T09:13:44,067 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T09:13:44,089 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4067 (auto-detected) 2024-12-03T09:13:44,090 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-03T09:13:44,106 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35781 2024-12-03T09:13:44,124 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35781 connecting to ZooKeeper ensemble=127.0.0.1:58590 2024-12-03T09:13:44,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:357810x0, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T09:13:44,240 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35781-0x1019a77fcb50000 connected 2024-12-03T09:13:44,341 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:13:44,345 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:13:44,356 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T09:13:44,359 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09, hbase.cluster.distributed=false 2024-12-03T09:13:44,378 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T09:13:44,382 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35781 2024-12-03T09:13:44,383 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35781 2024-12-03T09:13:44,383 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35781 2024-12-03T09:13:44,383 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35781 2024-12-03T09:13:44,384 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35781 2024-12-03T09:13:44,476 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/699c9baf3d29:0 server-side Connection retries=45 2024-12-03T09:13:44,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T09:13:44,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T09:13:44,478 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T09:13:44,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T09:13:44,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T09:13:44,480 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T09:13:44,482 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T09:13:44,483 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42271 2024-12-03T09:13:44,484 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42271 connecting to ZooKeeper ensemble=127.0.0.1:58590 2024-12-03T09:13:44,485 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:13:44,488 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:13:44,503 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422710x0, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T09:13:44,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:422710x0, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T09:13:44,504 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42271-0x1019a77fcb50001 connected 2024-12-03T09:13:44,507 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T09:13:44,515 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T09:13:44,518 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T09:13:44,523 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T09:13:44,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42271 2024-12-03T09:13:44,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42271 2024-12-03T09:13:44,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42271 2024-12-03T09:13:44,525 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42271 2024-12-03T09:13:44,525 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42271 2024-12-03T09:13:44,539 DEBUG [M:0;699c9baf3d29:35781 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;699c9baf3d29:35781 2024-12-03T09:13:44,540 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/699c9baf3d29,35781,1733217223715 2024-12-03T09:13:44,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T09:13:44,556 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T09:13:44,559 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/699c9baf3d29,35781,1733217223715 2024-12-03T09:13:44,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:13:44,587 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T09:13:44,587 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:13:44,588 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T09:13:44,589 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/699c9baf3d29,35781,1733217223715 from backup master directory 2024-12-03T09:13:44,602 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T09:13:44,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/699c9baf3d29,35781,1733217223715 2024-12-03T09:13:44,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T09:13:44,602 WARN [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T09:13:44,603 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=699c9baf3d29,35781,1733217223715 2024-12-03T09:13:44,605 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T09:13:44,607 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T09:13:44,654 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/hbase.id] with ID: c252829b-0c30-4211-bed0-58579dd285df 2024-12-03T09:13:44,654 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/.tmp/hbase.id 2024-12-03T09:13:44,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741826_1002 (size=42) 2024-12-03T09:13:45,069 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/.tmp/hbase.id]:[hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/hbase.id] 2024-12-03T09:13:45,113 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:13:45,117 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T09:13:45,136 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-12-03T09:13:45,192 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:13:45,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:13:45,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741827_1003 (size=196) 2024-12-03T09:13:45,634 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T09:13:45,637 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T09:13:45,651 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:45,655 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:13:45,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741828_1004 (size=1189) 2024-12-03T09:13:46,100 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store 2024-12-03T09:13:46,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741829_1005 (size=34) 2024-12-03T09:13:46,121 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T09:13:46,124 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:13:46,125 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T09:13:46,125 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:13:46,126 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:13:46,127 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T09:13:46,128 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:13:46,128 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:13:46,130 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733217226125Disabling compacts and flushes for region at 1733217226125Disabling writes for close at 1733217226128 (+3 ms)Writing region close event to WAL at 1733217226128Closed at 1733217226128 2024-12-03T09:13:46,132 WARN [master/699c9baf3d29:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/.initializing 2024-12-03T09:13:46,132 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/WALs/699c9baf3d29,35781,1733217223715 2024-12-03T09:13:46,140 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:13:46,154 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=699c9baf3d29%2C35781%2C1733217223715, suffix=, logDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/WALs/699c9baf3d29,35781,1733217223715, archiveDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/oldWALs, maxLogs=10 2024-12-03T09:13:46,174 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/WALs/699c9baf3d29,35781,1733217223715/699c9baf3d29%2C35781%2C1733217223715.1733217226158, exclude list is [], retry=0 2024-12-03T09:13:46,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39265,DS-54ab912e-b045-4c7b-b806-dfaa9a0f3e78,DISK] 2024-12-03T09:13:46,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T09:13:46,220 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/WALs/699c9baf3d29,35781,1733217223715/699c9baf3d29%2C35781%2C1733217223715.1733217226158 2024-12-03T09:13:46,221 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36147:36147)] 2024-12-03T09:13:46,222 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:13:46,222 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:13:46,225 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:13:46,226 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:13:46,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:13:46,278 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T09:13:46,281 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:46,283 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:13:46,284 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:13:46,287 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T09:13:46,287 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:46,288 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:13:46,288 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:13:46,290 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T09:13:46,291 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:46,291 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:13:46,292 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:13:46,294 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T09:13:46,294 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:46,295 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:13:46,295 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:13:46,298 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:13:46,299 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:13:46,304 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:13:46,305 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:13:46,308 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T09:13:46,312 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:13:46,317 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:13:46,318 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60219824, jitterRate=-0.10265469551086426}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T09:13:46,323 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733217226236Initializing all the Stores at 1733217226238 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217226239 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217226239Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217226239Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217226239Cleaning up temporary data from old regions at 1733217226305 (+66 ms)Region opened successfully at 1733217226322 (+17 ms) 2024-12-03T09:13:46,324 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T09:13:46,352 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f113b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=699c9baf3d29/172.17.0.2:0 2024-12-03T09:13:46,379 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T09:13:46,389 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T09:13:46,389 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T09:13:46,392 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T09:13:46,394 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T09:13:46,398 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T09:13:46,398 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T09:13:46,420 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T09:13:46,428 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T09:13:46,497 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T09:13:46,502 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T09:13:46,505 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T09:13:46,517 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T09:13:46,521 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T09:13:46,525 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T09:13:46,538 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T09:13:46,541 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T09:13:46,549 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T09:13:46,570 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T09:13:46,580 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T09:13:46,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T09:13:46,591 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T09:13:46,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:13:46,591 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:13:46,595 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=699c9baf3d29,35781,1733217223715, sessionid=0x1019a77fcb50000, setting cluster-up flag (Was=false) 2024-12-03T09:13:46,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:13:46,629 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:13:46,665 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T09:13:46,671 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=699c9baf3d29,35781,1733217223715 2024-12-03T09:13:46,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:13:46,696 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:13:46,728 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T09:13:46,732 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=699c9baf3d29,35781,1733217223715 2024-12-03T09:13:46,740 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T09:13:46,806 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T09:13:46,815 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T09:13:46,821 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T09:13:46,826 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 699c9baf3d29,35781,1733217223715 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T09:13:46,831 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(746): ClusterId : c252829b-0c30-4211-bed0-58579dd285df 2024-12-03T09:13:46,833 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/699c9baf3d29:0, corePoolSize=5, maxPoolSize=5 2024-12-03T09:13:46,833 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/699c9baf3d29:0, corePoolSize=5, maxPoolSize=5 2024-12-03T09:13:46,833 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T09:13:46,833 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/699c9baf3d29:0, corePoolSize=5, maxPoolSize=5 2024-12-03T09:13:46,834 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/699c9baf3d29:0, corePoolSize=5, maxPoolSize=5 2024-12-03T09:13:46,834 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/699c9baf3d29:0, corePoolSize=10, maxPoolSize=10 2024-12-03T09:13:46,834 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:13:46,834 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/699c9baf3d29:0, corePoolSize=2, maxPoolSize=2 2024-12-03T09:13:46,834 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:13:46,835 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733217256835 2024-12-03T09:13:46,837 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T09:13:46,837 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T09:13:46,839 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T09:13:46,840 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T09:13:46,841 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T09:13:46,841 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T09:13:46,842 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T09:13:46,842 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T09:13:46,842 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:46,846 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T09:13:46,846 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:46,846 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T09:13:46,846 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T09:13:46,846 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T09:13:46,847 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T09:13:46,847 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T09:13:46,849 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T09:13:46,849 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T09:13:46,851 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/699c9baf3d29:0:becomeActiveMaster-HFileCleaner.large.0-1733217226850,5,FailOnTimeoutGroup] 2024-12-03T09:13:46,851 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/699c9baf3d29:0:becomeActiveMaster-HFileCleaner.small.0-1733217226851,5,FailOnTimeoutGroup] 2024-12-03T09:13:46,851 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:46,851 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T09:13:46,852 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:46,853 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:46,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741831_1007 (size=1321) 2024-12-03T09:13:46,855 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T09:13:46,856 DEBUG [RS:0;699c9baf3d29:42271 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@652b315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=699c9baf3d29/172.17.0.2:0 2024-12-03T09:13:46,868 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;699c9baf3d29:42271 2024-12-03T09:13:46,871 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T09:13:46,871 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T09:13:46,871 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T09:13:46,873 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(2659): reportForDuty to master=699c9baf3d29,35781,1733217223715 with port=42271, startcode=1733217224446 2024-12-03T09:13:46,882 DEBUG [RS:0;699c9baf3d29:42271 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T09:13:46,914 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52003, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T09:13:46,919 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35781 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 699c9baf3d29,42271,1733217224446 2024-12-03T09:13:46,921 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35781 {}] master.ServerManager(517): Registering regionserver=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:46,934 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09 2024-12-03T09:13:46,934 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43771 2024-12-03T09:13:46,934 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T09:13:46,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T09:13:46,950 DEBUG [RS:0;699c9baf3d29:42271 {}] zookeeper.ZKUtil(111): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/699c9baf3d29,42271,1733217224446 2024-12-03T09:13:46,950 WARN [RS:0;699c9baf3d29:42271 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T09:13:46,950 INFO [RS:0;699c9baf3d29:42271 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:13:46,950 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/WALs/699c9baf3d29,42271,1733217224446 2024-12-03T09:13:46,952 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [699c9baf3d29,42271,1733217224446] 2024-12-03T09:13:46,974 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T09:13:46,985 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T09:13:46,989 INFO [RS:0;699c9baf3d29:42271 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T09:13:46,989 INFO [RS:0;699c9baf3d29:42271 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:46,990 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T09:13:46,995 INFO [RS:0;699c9baf3d29:42271 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T09:13:46,996 INFO [RS:0;699c9baf3d29:42271 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:46,996 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:13:46,996 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:13:46,996 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:13:46,997 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:13:46,997 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:13:46,997 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0, corePoolSize=2, maxPoolSize=2 2024-12-03T09:13:46,997 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:13:46,997 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:13:46,997 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:13:46,997 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:13:46,997 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:13:46,998 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:13:46,998 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/699c9baf3d29:0, corePoolSize=3, maxPoolSize=3 2024-12-03T09:13:46,998 DEBUG [RS:0;699c9baf3d29:42271 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0, corePoolSize=3, maxPoolSize=3 2024-12-03T09:13:46,999 INFO [RS:0;699c9baf3d29:42271 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:46,999 INFO [RS:0;699c9baf3d29:42271 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:46,999 INFO [RS:0;699c9baf3d29:42271 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:46,999 INFO [RS:0;699c9baf3d29:42271 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:46,999 INFO [RS:0;699c9baf3d29:42271 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:46,999 INFO [RS:0;699c9baf3d29:42271 {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,42271,1733217224446-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T09:13:47,016 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T09:13:47,017 INFO [RS:0;699c9baf3d29:42271 {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,42271,1733217224446-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:47,018 INFO [RS:0;699c9baf3d29:42271 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:47,018 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.Replication(171): 699c9baf3d29,42271,1733217224446 started 2024-12-03T09:13:47,033 INFO [RS:0;699c9baf3d29:42271 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:47,033 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1482): Serving as 699c9baf3d29,42271,1733217224446, RpcServer on 699c9baf3d29/172.17.0.2:42271, sessionid=0x1019a77fcb50001 2024-12-03T09:13:47,034 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T09:13:47,034 DEBUG [RS:0;699c9baf3d29:42271 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 699c9baf3d29,42271,1733217224446 2024-12-03T09:13:47,034 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '699c9baf3d29,42271,1733217224446' 2024-12-03T09:13:47,034 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T09:13:47,035 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T09:13:47,036 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T09:13:47,036 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T09:13:47,036 DEBUG [RS:0;699c9baf3d29:42271 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 699c9baf3d29,42271,1733217224446 2024-12-03T09:13:47,036 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '699c9baf3d29,42271,1733217224446' 2024-12-03T09:13:47,036 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T09:13:47,037 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T09:13:47,038 DEBUG [RS:0;699c9baf3d29:42271 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T09:13:47,038 INFO [RS:0;699c9baf3d29:42271 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T09:13:47,038 INFO [RS:0;699c9baf3d29:42271 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T09:13:47,149 INFO [RS:0;699c9baf3d29:42271 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:13:47,153 INFO [RS:0;699c9baf3d29:42271 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=699c9baf3d29%2C42271%2C1733217224446, suffix=, logDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/WALs/699c9baf3d29,42271,1733217224446, archiveDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/oldWALs, maxLogs=32 2024-12-03T09:13:47,165 DEBUG [RS:0;699c9baf3d29:42271 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/WALs/699c9baf3d29,42271,1733217224446/699c9baf3d29%2C42271%2C1733217224446.1733217227155, exclude list is [], retry=0 2024-12-03T09:13:47,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39265,DS-54ab912e-b045-4c7b-b806-dfaa9a0f3e78,DISK] 2024-12-03T09:13:47,173 INFO [RS:0;699c9baf3d29:42271 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/WALs/699c9baf3d29,42271,1733217224446/699c9baf3d29%2C42271%2C1733217224446.1733217227155 2024-12-03T09:13:47,174 DEBUG [RS:0;699c9baf3d29:42271 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36147:36147)] 2024-12-03T09:13:47,259 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T09:13:47,260 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09 2024-12-03T09:13:47,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741833_1009 (size=32) 2024-12-03T09:13:47,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:13:47,280 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T09:13:47,283 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T09:13:47,283 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:47,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:13:47,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T09:13:47,288 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T09:13:47,288 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:47,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:13:47,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T09:13:47,293 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T09:13:47,293 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:47,294 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:13:47,294 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T09:13:47,297 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T09:13:47,297 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:47,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:13:47,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T09:13:47,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740 2024-12-03T09:13:47,300 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740 2024-12-03T09:13:47,302 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T09:13:47,303 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T09:13:47,303 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T09:13:47,305 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T09:13:47,309 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:13:47,309 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70971474, jitterRate=0.057557374238967896}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T09:13:47,312 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733217227277Initializing all the Stores at 1733217227279 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217227279Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217227280 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217227280Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217227280Cleaning up temporary data from old regions at 1733217227303 (+23 ms)Region opened successfully at 1733217227312 (+9 ms) 2024-12-03T09:13:47,312 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T09:13:47,312 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T09:13:47,312 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T09:13:47,312 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T09:13:47,312 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T09:13:47,313 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T09:13:47,314 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733217227312Disabling compacts and flushes for region at 1733217227312Disabling writes for close at 1733217227312Writing region close event to WAL at 1733217227313 (+1 ms)Closed at 1733217227313 2024-12-03T09:13:47,317 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T09:13:47,317 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T09:13:47,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T09:13:47,329 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T09:13:47,332 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T09:13:47,486 DEBUG [699c9baf3d29:35781 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T09:13:47,501 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:47,506 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 699c9baf3d29,42271,1733217224446, state=OPENING 2024-12-03T09:13:47,560 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T09:13:47,570 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:13:47,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:13:47,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:13:47,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:13:47,575 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T09:13:47,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:13:47,755 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T09:13:47,758 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32981, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T09:13:47,768 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T09:13:47,768 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:13:47,769 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T09:13:47,772 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=699c9baf3d29%2C42271%2C1733217224446.meta, suffix=.meta, logDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/WALs/699c9baf3d29,42271,1733217224446, archiveDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/oldWALs, maxLogs=32 2024-12-03T09:13:47,787 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/WALs/699c9baf3d29,42271,1733217224446/699c9baf3d29%2C42271%2C1733217224446.meta.1733217227774.meta, exclude list is [], retry=0 2024-12-03T09:13:47,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39265,DS-54ab912e-b045-4c7b-b806-dfaa9a0f3e78,DISK] 2024-12-03T09:13:47,793 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/WALs/699c9baf3d29,42271,1733217224446/699c9baf3d29%2C42271%2C1733217224446.meta.1733217227774.meta 2024-12-03T09:13:47,793 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36147:36147)] 2024-12-03T09:13:47,794 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:13:47,795 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T09:13:47,797 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T09:13:47,801 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T09:13:47,805 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T09:13:47,806 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:13:47,806 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T09:13:47,806 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T09:13:47,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T09:13:47,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T09:13:47,811 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:47,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:13:47,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T09:13:47,813 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T09:13:47,813 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:47,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:13:47,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T09:13:47,815 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T09:13:47,815 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:47,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:13:47,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T09:13:47,818 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T09:13:47,818 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:47,819 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:13:47,819 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T09:13:47,820 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740 2024-12-03T09:13:47,822 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740 2024-12-03T09:13:47,825 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T09:13:47,825 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T09:13:47,826 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T09:13:47,828 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T09:13:47,830 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67967874, jitterRate=0.012800246477127075}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T09:13:47,830 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T09:13:47,831 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733217227806Writing region info on filesystem at 1733217227807 (+1 ms)Initializing all the Stores at 1733217227809 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217227809Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217227809Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217227809Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217227809Cleaning up temporary data from old regions at 1733217227825 (+16 ms)Running coprocessor post-open hooks at 1733217227830 (+5 ms)Region opened successfully at 1733217227831 (+1 ms) 2024-12-03T09:13:47,837 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733217227745 2024-12-03T09:13:47,846 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T09:13:47,847 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T09:13:47,848 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:47,849 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 699c9baf3d29,42271,1733217224446, state=OPEN 2024-12-03T09:13:47,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T09:13:47,891 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T09:13:47,891 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:13:47,891 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:13:47,892 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:47,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T09:13:47,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=699c9baf3d29,42271,1733217224446 in 315 msec 2024-12-03T09:13:47,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T09:13:47,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 580 msec 2024-12-03T09:13:47,910 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T09:13:47,910 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T09:13:47,929 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:13:47,930 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:13:47,947 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:13:47,949 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47059, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:13:47,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1990 sec 2024-12-03T09:13:47,968 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733217227968, completionTime=-1 2024-12-03T09:13:47,971 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T09:13:47,971 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T09:13:47,993 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T09:13:47,993 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733217287993 2024-12-03T09:13:47,994 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733217347993 2024-12-03T09:13:47,994 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-12-03T09:13:47,996 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,35781,1733217223715-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:47,996 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,35781,1733217223715-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:47,996 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,35781,1733217223715-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:47,998 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-699c9baf3d29:35781, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:47,998 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:47,999 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:48,004 DEBUG [master/699c9baf3d29:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T09:13:48,025 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.422sec 2024-12-03T09:13:48,026 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T09:13:48,027 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T09:13:48,028 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T09:13:48,029 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T09:13:48,029 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T09:13:48,029 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,35781,1733217223715-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T09:13:48,030 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,35781,1733217223715-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T09:13:48,039 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T09:13:48,040 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T09:13:48,040 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,35781,1733217223715-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:13:48,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d3128f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:48,042 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T09:13:48,042 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T09:13:48,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:13:48,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:13:48,053 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:13:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:13:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:13:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db5a53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:13:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:13:48,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:13:48,063 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:13:48,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c08b87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:48,066 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:13:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:13:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:13:48,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:13:48,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=699c9baf3d29,35781,1733217223715 2024-12-03T09:13:48,102 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=229, ProcessCount=11, AvailableMemoryMB=13714 2024-12-03T09:13:48,110 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T09:13:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 699c9baf3d29,35781,1733217223715 2024-12-03T09:13:48,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f523d87 2024-12-03T09:13:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T09:13:48,134 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T09:13:48,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T09:13:48,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T09:13:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T09:13:48,159 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T09:13:48,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-03T09:13:48,163 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:48,165 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T09:13:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T09:13:48,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741835_1011 (size=1061) 2024-12-03T09:13:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T09:13:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T09:13:48,586 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09 2024-12-03T09:13:48,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741836_1012 (size=53) 2024-12-03T09:13:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T09:13:48,998 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:13:48,998 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c49f86fc80f2af2d2b0cf03332e8cc12, disabling compactions & flushes 2024-12-03T09:13:48,998 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:48,998 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:48,998 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. after waiting 0 ms 2024-12-03T09:13:48,998 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:48,998 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:48,999 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c49f86fc80f2af2d2b0cf03332e8cc12: Waiting for close lock at 1733217228998Disabling compacts and flushes for region at 1733217228998Disabling writes for close at 1733217228998Writing region close event to WAL at 1733217228998Closed at 1733217228998 2024-12-03T09:13:49,001 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T09:13:49,008 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733217229001"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733217229001"}]},"ts":"1733217229001"} 2024-12-03T09:13:49,013 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T09:13:49,015 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T09:13:49,017 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217229015"}]},"ts":"1733217229015"} 2024-12-03T09:13:49,021 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T09:13:49,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49f86fc80f2af2d2b0cf03332e8cc12, ASSIGN}] 2024-12-03T09:13:49,024 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49f86fc80f2af2d2b0cf03332e8cc12, ASSIGN 2024-12-03T09:13:49,026 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49f86fc80f2af2d2b0cf03332e8cc12, ASSIGN; state=OFFLINE, location=699c9baf3d29,42271,1733217224446; forceNewPlan=false, retain=false 2024-12-03T09:13:49,177 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c49f86fc80f2af2d2b0cf03332e8cc12, regionState=OPENING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:49,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49f86fc80f2af2d2b0cf03332e8cc12, ASSIGN because future has completed 2024-12-03T09:13:49,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:13:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T09:13:49,350 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:49,350 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:13:49,351 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:49,351 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:13:49,351 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:49,351 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:49,353 INFO [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:49,356 INFO [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:13:49,357 INFO [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c49f86fc80f2af2d2b0cf03332e8cc12 columnFamilyName A 2024-12-03T09:13:49,357 DEBUG [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:49,358 INFO [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] regionserver.HStore(327): Store=c49f86fc80f2af2d2b0cf03332e8cc12/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:13:49,359 INFO [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:49,361 INFO [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:13:49,361 INFO [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c49f86fc80f2af2d2b0cf03332e8cc12 columnFamilyName B 2024-12-03T09:13:49,361 DEBUG [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:49,362 INFO [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] regionserver.HStore(327): Store=c49f86fc80f2af2d2b0cf03332e8cc12/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:13:49,362 INFO [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:49,364 INFO [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:13:49,364 INFO [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c49f86fc80f2af2d2b0cf03332e8cc12 columnFamilyName C 2024-12-03T09:13:49,365 DEBUG [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:13:49,365 INFO [StoreOpener-c49f86fc80f2af2d2b0cf03332e8cc12-1 {}] regionserver.HStore(327): Store=c49f86fc80f2af2d2b0cf03332e8cc12/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:13:49,366 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:49,366 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:49,367 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:49,368 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:49,370 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:49,370 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:49,371 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T09:13:49,373 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:49,376 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:13:49,377 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c49f86fc80f2af2d2b0cf03332e8cc12; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62384861, jitterRate=-0.07039313018321991}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T09:13:49,377 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:49,378 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c49f86fc80f2af2d2b0cf03332e8cc12: Running coprocessor pre-open hook at 1733217229351Writing region info on filesystem at 1733217229351Initializing all the Stores at 1733217229353 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217229353Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217229353Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217229353Cleaning up temporary data from old regions at 1733217229370 (+17 ms)Running coprocessor post-open hooks at 1733217229377 (+7 ms)Region opened successfully at 1733217229378 (+1 ms) 2024-12-03T09:13:49,380 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., pid=6, masterSystemTime=1733217229338 2024-12-03T09:13:49,384 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:49,384 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:49,385 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c49f86fc80f2af2d2b0cf03332e8cc12, regionState=OPEN, openSeqNum=2, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:49,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:13:49,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T09:13:49,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 in 208 msec 2024-12-03T09:13:49,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T09:13:49,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49f86fc80f2af2d2b0cf03332e8cc12, ASSIGN in 372 msec 2024-12-03T09:13:49,400 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T09:13:49,400 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217229400"}]},"ts":"1733217229400"} 2024-12-03T09:13:49,403 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T09:13:49,405 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T09:13:49,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2590 sec 2024-12-03T09:13:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T09:13:50,331 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:13:50,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44898a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:13:50,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:13:50,340 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:13:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:13:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:13:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cd3e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:13:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:13:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:13:50,344 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:13:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c82fed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,346 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:13:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:13:50,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:13:50,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:13:50,353 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T09:13:50,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 699c9baf3d29,35781,1733217223715 2024-12-03T09:13:50,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@da7d497 2024-12-03T09:13:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T09:13:50,359 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T09:13:50,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@265b3055, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,363 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:13:50,363 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:13:50,365 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:13:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:13:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:13:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:13:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:13:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:13:50,368 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:13:50,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@776dcc7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,370 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:13:50,370 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:13:50,372 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:13:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:13:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:13:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd81cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:13:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:13:50,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:13:50,375 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:13:50,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a6e4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b83180b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:13:50,378 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:13:50,380 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:13:50,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:13:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:13:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39deefb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:13:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:13:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:13:50,383 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:13:50,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ac8568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6167b64c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:13:50,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:13:50,389 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:13:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:13:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:13:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f535689, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:13:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:13:50,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:13:50,392 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:13:50,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bffd9e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@504dd62a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:13:50,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:13:50,395 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:13:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:13:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:13:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61fba9b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:13:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:13:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:13:50,398 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:13:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5904dd5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d8c4061, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,401 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:13:50,401 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:13:50,402 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:13:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:13:50,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:13:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b036a61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:13:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:13:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:13:50,405 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:13:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129161b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16aad90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:13:50,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:13:50,409 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:13:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:13:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:13:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d111af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:13:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:13:50,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:13:50,412 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:13:50,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e9639cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3461ebf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:13:50,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:13:50,417 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:13:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:13:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:13:50,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e406351, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:13:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:13:50,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:13:50,419 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:13:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d22a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29322cfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:13:50,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:13:50,423 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:13:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:13:50,423 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:13:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17ddb883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:13:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:13:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:13:50,425 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:13:50,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2171b18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:13:50,432 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:13:50,432 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:13:50,432 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:13:50,433 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:13:50,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:13:50,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:13:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:13:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:13:50,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:13:50,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:13:50,437 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:13:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:13:50,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:13:50,438 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:13:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:13:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:13:50,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:13:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:13:50,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:13:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-03T09:13:50,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:13:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:13:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:13:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:13:50,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:13:50,451 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:13:50,453 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:13:50,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T09:13:50,454 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:13:50,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:13:50,455 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:13:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:13:50,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:13:50,461 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:13:50,462 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:13:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:13:50,464 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:13:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:13:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:13:50,466 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:13:50,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:13:50,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:13:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:13:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:13:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:13:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:13:50,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:13:50,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:13:50,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:13:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:13:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:13:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:50,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:13:50,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:13:50,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:50,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:13:50,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:50,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:13:50,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:50,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T09:13:50,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T09:13:50,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:50,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/5d9367b189674ce498fa2b9dd0a7b39c is 50, key is test_row_0/A:col10/1733217230514/Put/seqid=0 2024-12-03T09:13:50,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:50,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:50,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:50,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741837_1013 (size=12001) 2024-12-03T09:13:50,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:50,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/5d9367b189674ce498fa2b9dd0a7b39c 2024-12-03T09:13:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217290644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:50,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217290645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:50,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:50,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217290656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217290671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:50,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217290671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:50,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T09:13:50,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/374b98ebc5004100b844e84868c73d75 is 50, key is test_row_0/B:col10/1733217230514/Put/seqid=0 2024-12-03T09:13:50,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217290811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:50,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217290811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:50,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:50,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:50,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217290812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217290812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741838_1014 (size=12001) 2024-12-03T09:13:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:50,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/374b98ebc5004100b844e84868c73d75 2024-12-03T09:13:50,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T09:13:50,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217290813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:50,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:50,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:50,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:50,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:50,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:50,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/b1dc583720c04e138f0ecf6c9e61c804 is 50, key is test_row_0/C:col10/1733217230514/Put/seqid=0 2024-12-03T09:13:50,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741839_1015 (size=12001) 2024-12-03T09:13:50,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/b1dc583720c04e138f0ecf6c9e61c804 2024-12-03T09:13:50,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/5d9367b189674ce498fa2b9dd0a7b39c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/5d9367b189674ce498fa2b9dd0a7b39c 2024-12-03T09:13:50,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/5d9367b189674ce498fa2b9dd0a7b39c, entries=150, sequenceid=13, filesize=11.7 K 2024-12-03T09:13:50,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/374b98ebc5004100b844e84868c73d75 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/374b98ebc5004100b844e84868c73d75 2024-12-03T09:13:50,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/374b98ebc5004100b844e84868c73d75, entries=150, sequenceid=13, filesize=11.7 K 2024-12-03T09:13:50,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/b1dc583720c04e138f0ecf6c9e61c804 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b1dc583720c04e138f0ecf6c9e61c804 2024-12-03T09:13:50,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b1dc583720c04e138f0ecf6c9e61c804, entries=150, sequenceid=13, filesize=11.7 K 2024-12-03T09:13:51,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T09:13:51,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c49f86fc80f2af2d2b0cf03332e8cc12 in 471ms, sequenceid=13, compaction requested=false 2024-12-03T09:13:51,005 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T09:13:51,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:51,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:51,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:51,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:51,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:51,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:51,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217230589, max timestamp=1733217231021Num uniques 859; 2024-12-03T09:13:51,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:13:51,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217230589, max timestamp=1733217231021Num uniques 859; 2024-12-03T09:13:51,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:13:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:51,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=71.56 KB heapSize=204.02 KB 2024-12-03T09:13:51,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:13:51,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:13:51,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:13:51,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:13:51,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:13:51,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:13:51,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/3e96c59532b14925a1f6f37cd3340905 is 50, key is test_row_0/A:col10/1733217230635/Put/seqid=0 2024-12-03T09:13:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T09:13:51,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741840_1016 (size=12001) 2024-12-03T09:13:51,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/3e96c59532b14925a1f6f37cd3340905 2024-12-03T09:13:51,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/b655b1253dbd4e5e965f2e545aac2bfc is 50, key is test_row_0/B:col10/1733217230635/Put/seqid=0 2024-12-03T09:13:51,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217291124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217291128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217291129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217291133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217291133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741841_1017 (size=12001) 2024-12-03T09:13:51,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/b655b1253dbd4e5e965f2e545aac2bfc 2024-12-03T09:13:51,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T09:13:51,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:51,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:51,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:51,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:51,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:51,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/bcbd408323b74df0ba383f2e736237b8 is 50, key is test_row_0/C:col10/1733217230635/Put/seqid=0 2024-12-03T09:13:51,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741842_1018 (size=12001) 2024-12-03T09:13:51,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/bcbd408323b74df0ba383f2e736237b8 2024-12-03T09:13:51,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/3e96c59532b14925a1f6f37cd3340905 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/3e96c59532b14925a1f6f37cd3340905 2024-12-03T09:13:51,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217291249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217291250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217291250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217291251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217291261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/3e96c59532b14925a1f6f37cd3340905, entries=150, sequenceid=36, filesize=11.7 K 2024-12-03T09:13:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/b655b1253dbd4e5e965f2e545aac2bfc as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/b655b1253dbd4e5e965f2e545aac2bfc 2024-12-03T09:13:51,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/b655b1253dbd4e5e965f2e545aac2bfc, entries=150, sequenceid=36, filesize=11.7 K 2024-12-03T09:13:51,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/bcbd408323b74df0ba383f2e736237b8 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/bcbd408323b74df0ba383f2e736237b8 2024-12-03T09:13:51,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/bcbd408323b74df0ba383f2e736237b8, entries=150, sequenceid=36, filesize=11.7 K 2024-12-03T09:13:51,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~203.97 KB/208864, currentSize=114.05 KB/116790 for c49f86fc80f2af2d2b0cf03332e8cc12 in 261ms, sequenceid=36, compaction requested=false 2024-12-03T09:13:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:51,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T09:13:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:51,323 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T09:13:51,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:13:51,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:51,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:13:51,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:51,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:13:51,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:51,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/2bbefc43af994b148d278904fee16500 is 50, key is test_row_0/A:col10/1733217231127/Put/seqid=0 2024-12-03T09:13:51,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741843_1019 (size=12001) 2024-12-03T09:13:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:51,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:51,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217291492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217291493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217291495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217291497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217291501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T09:13:51,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217291611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217291611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217291611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217291612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217291620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,756 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/2bbefc43af994b148d278904fee16500 2024-12-03T09:13:51,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/574ff23901c447fda67b6d6c32e6011b is 50, key is test_row_0/B:col10/1733217231127/Put/seqid=0 2024-12-03T09:13:51,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217291820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217291820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217291820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741844_1020 (size=12001) 2024-12-03T09:13:51,836 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/574ff23901c447fda67b6d6c32e6011b 2024-12-03T09:13:51,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217291830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217291830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:51,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/9a48c8f681f946aaa19de8ab6948e60c is 50, key is test_row_0/C:col10/1733217231127/Put/seqid=0 2024-12-03T09:13:51,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741845_1021 (size=12001) 2024-12-03T09:13:52,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217292129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217292140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217292140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217292150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217292159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,292 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/9a48c8f681f946aaa19de8ab6948e60c 2024-12-03T09:13:52,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/2bbefc43af994b148d278904fee16500 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2bbefc43af994b148d278904fee16500 2024-12-03T09:13:52,329 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2bbefc43af994b148d278904fee16500, entries=150, sequenceid=56, filesize=11.7 K 2024-12-03T09:13:52,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/574ff23901c447fda67b6d6c32e6011b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/574ff23901c447fda67b6d6c32e6011b 2024-12-03T09:13:52,348 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/574ff23901c447fda67b6d6c32e6011b, entries=150, sequenceid=56, filesize=11.7 K 2024-12-03T09:13:52,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/9a48c8f681f946aaa19de8ab6948e60c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/9a48c8f681f946aaa19de8ab6948e60c 2024-12-03T09:13:52,370 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/9a48c8f681f946aaa19de8ab6948e60c, entries=150, sequenceid=56, filesize=11.7 K 2024-12-03T09:13:52,372 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for c49f86fc80f2af2d2b0cf03332e8cc12 in 1049ms, sequenceid=56, compaction requested=true 2024-12-03T09:13:52,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:52,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:52,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-03T09:13:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-03T09:13:52,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T09:13:52,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9100 sec 2024-12-03T09:13:52,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9450 sec 2024-12-03T09:13:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T09:13:52,600 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:13:52,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:13:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-03T09:13:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T09:13:52,606 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:13:52,609 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:13:52,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:13:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:52,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T09:13:52,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:13:52,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:52,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:13:52,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:52,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:13:52,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:52,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/0ead33cad4af426ab4848954c282b161 is 50, key is test_row_0/A:col10/1733217231494/Put/seqid=0 2024-12-03T09:13:52,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217292694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217292691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217292697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217292699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T09:13:52,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217292704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741846_1022 (size=12001) 2024-12-03T09:13:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/0ead33cad4af426ab4848954c282b161 2024-12-03T09:13:52,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/43565553c69a4f42941d60c674cd0335 is 50, key is test_row_0/B:col10/1733217231494/Put/seqid=0 2024-12-03T09:13:52,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741847_1023 (size=12001) 2024-12-03T09:13:52,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/43565553c69a4f42941d60c674cd0335 2024-12-03T09:13:52,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T09:13:52,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:52,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:52,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/a21d0c3a756f49f0941b177179c8a89b is 50, key is test_row_0/C:col10/1733217231494/Put/seqid=0 2024-12-03T09:13:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:52,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:52,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741848_1024 (size=12001) 2024-12-03T09:13:52,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/a21d0c3a756f49f0941b177179c8a89b 2024-12-03T09:13:52,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217292810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217292810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217292812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/0ead33cad4af426ab4848954c282b161 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/0ead33cad4af426ab4848954c282b161 2024-12-03T09:13:52,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217292820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217292820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:52,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/0ead33cad4af426ab4848954c282b161, entries=150, sequenceid=70, filesize=11.7 K 2024-12-03T09:13:52,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/43565553c69a4f42941d60c674cd0335 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/43565553c69a4f42941d60c674cd0335 2024-12-03T09:13:52,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/43565553c69a4f42941d60c674cd0335, entries=150, sequenceid=70, filesize=11.7 K 2024-12-03T09:13:52,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/a21d0c3a756f49f0941b177179c8a89b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/a21d0c3a756f49f0941b177179c8a89b 2024-12-03T09:13:52,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/a21d0c3a756f49f0941b177179c8a89b, entries=150, sequenceid=70, filesize=11.7 K 2024-12-03T09:13:52,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c49f86fc80f2af2d2b0cf03332e8cc12 in 226ms, sequenceid=70, compaction requested=true 2024-12-03T09:13:52,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:52,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:13:52,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:13:52,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:13:52,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:13:52,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:13:52,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:13:52,881 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:13:52,881 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:13:52,892 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:13:52,894 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/B is initiating minor compaction (all files) 2024-12-03T09:13:52,895 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/B in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:52,896 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/374b98ebc5004100b844e84868c73d75, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/b655b1253dbd4e5e965f2e545aac2bfc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/574ff23901c447fda67b6d6c32e6011b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/43565553c69a4f42941d60c674cd0335] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=46.9 K 2024-12-03T09:13:52,898 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 374b98ebc5004100b844e84868c73d75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733217230485 2024-12-03T09:13:52,899 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting b655b1253dbd4e5e965f2e545aac2bfc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733217230635 2024-12-03T09:13:52,901 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 574ff23901c447fda67b6d6c32e6011b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733217231109 2024-12-03T09:13:52,910 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:13:52,910 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 43565553c69a4f42941d60c674cd0335, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733217231488 2024-12-03T09:13:52,910 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/A is initiating minor compaction (all files) 2024-12-03T09:13:52,910 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/A in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:52,911 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/5d9367b189674ce498fa2b9dd0a7b39c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/3e96c59532b14925a1f6f37cd3340905, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2bbefc43af994b148d278904fee16500, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/0ead33cad4af426ab4848954c282b161] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=46.9 K 2024-12-03T09:13:52,913 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d9367b189674ce498fa2b9dd0a7b39c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733217230485 2024-12-03T09:13:52,914 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e96c59532b14925a1f6f37cd3340905, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733217230635 2024-12-03T09:13:52,916 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bbefc43af994b148d278904fee16500, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733217231109 2024-12-03T09:13:52,918 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ead33cad4af426ab4848954c282b161, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733217231488 2024-12-03T09:13:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T09:13:52,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T09:13:52,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:52,933 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T09:13:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:13:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:13:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:13:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:52,980 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#B#compaction#12 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:13:52,980 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#A#compaction#13 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:13:52,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/60ed4bf82f0949569cced6aac2eb48b2 is 50, key is test_row_0/A:col10/1733217232699/Put/seqid=0 2024-12-03T09:13:52,982 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/1fe0d613d20c4dd6b6f72c541d34d79b is 50, key is test_row_0/A:col10/1733217231494/Put/seqid=0 2024-12-03T09:13:52,984 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/ac360c91b0584c24bcc8b011a4afd9b8 is 50, key is test_row_0/B:col10/1733217231494/Put/seqid=0 2024-12-03T09:13:53,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741849_1025 (size=12001) 2024-12-03T09:13:53,007 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/60ed4bf82f0949569cced6aac2eb48b2 2024-12-03T09:13:53,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741850_1026 (size=12139) 2024-12-03T09:13:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:53,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:53,027 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/1fe0d613d20c4dd6b6f72c541d34d79b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/1fe0d613d20c4dd6b6f72c541d34d79b 2024-12-03T09:13:53,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741851_1027 (size=12139) 2024-12-03T09:13:53,058 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/ac360c91b0584c24bcc8b011a4afd9b8 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/ac360c91b0584c24bcc8b011a4afd9b8 2024-12-03T09:13:53,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/7500b0684a234779bae58bb0ed1a16b9 is 50, key is test_row_0/B:col10/1733217232699/Put/seqid=0 2024-12-03T09:13:53,068 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/A of c49f86fc80f2af2d2b0cf03332e8cc12 into 1fe0d613d20c4dd6b6f72c541d34d79b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:13:53,068 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:53,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217293053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,071 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/A, priority=12, startTime=1733217232873; duration=0sec 2024-12-03T09:13:53,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,074 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:13:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217293056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,074 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:A 2024-12-03T09:13:53,075 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:13:53,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,077 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/B of c49f86fc80f2af2d2b0cf03332e8cc12 into ac360c91b0584c24bcc8b011a4afd9b8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:13:53,077 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:13:53,077 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:53,077 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/C is initiating minor compaction (all files) 2024-12-03T09:13:53,077 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/B, priority=12, startTime=1733217232879; duration=0sec 2024-12-03T09:13:53,077 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/C in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:53,078 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:13:53,078 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:B 2024-12-03T09:13:53,078 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b1dc583720c04e138f0ecf6c9e61c804, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/bcbd408323b74df0ba383f2e736237b8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/9a48c8f681f946aaa19de8ab6948e60c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/a21d0c3a756f49f0941b177179c8a89b] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=46.9 K 2024-12-03T09:13:53,079 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1dc583720c04e138f0ecf6c9e61c804, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733217230485 2024-12-03T09:13:53,080 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcbd408323b74df0ba383f2e736237b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733217230635 2024-12-03T09:13:53,084 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a48c8f681f946aaa19de8ab6948e60c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733217231109 2024-12-03T09:13:53,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217293070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217293074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,087 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a21d0c3a756f49f0941b177179c8a89b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733217231488 2024-12-03T09:13:53,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217293074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741852_1028 (size=12001) 2024-12-03T09:13:53,117 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/7500b0684a234779bae58bb0ed1a16b9 2024-12-03T09:13:53,134 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#C#compaction#16 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:13:53,135 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/ac4e9bce9a2b47508aad69922e29a0de is 50, key is test_row_0/C:col10/1733217231494/Put/seqid=0 2024-12-03T09:13:53,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/adcf3681a6ad4a0bba3d253ebb86e6bf is 50, key is test_row_0/C:col10/1733217232699/Put/seqid=0 2024-12-03T09:13:53,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741853_1029 (size=12139) 2024-12-03T09:13:53,185 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/ac4e9bce9a2b47508aad69922e29a0de as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/ac4e9bce9a2b47508aad69922e29a0de 2024-12-03T09:13:53,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217293179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217293180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217293191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217293192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217293200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,203 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/C of c49f86fc80f2af2d2b0cf03332e8cc12 into ac4e9bce9a2b47508aad69922e29a0de(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:13:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,203 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:53,203 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/C, priority=12, startTime=1733217232879; duration=0sec 2024-12-03T09:13:53,204 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:13:53,204 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:C 2024-12-03T09:13:53,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741854_1030 (size=12001) 2024-12-03T09:13:53,207 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/adcf3681a6ad4a0bba3d253ebb86e6bf 2024-12-03T09:13:53,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/60ed4bf82f0949569cced6aac2eb48b2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/60ed4bf82f0949569cced6aac2eb48b2 2024-12-03T09:13:53,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T09:13:53,232 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/60ed4bf82f0949569cced6aac2eb48b2, entries=150, sequenceid=88, filesize=11.7 K 2024-12-03T09:13:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/7500b0684a234779bae58bb0ed1a16b9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/7500b0684a234779bae58bb0ed1a16b9 2024-12-03T09:13:53,249 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/7500b0684a234779bae58bb0ed1a16b9, entries=150, sequenceid=88, filesize=11.7 K 2024-12-03T09:13:53,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/adcf3681a6ad4a0bba3d253ebb86e6bf as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/adcf3681a6ad4a0bba3d253ebb86e6bf 2024-12-03T09:13:53,267 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/adcf3681a6ad4a0bba3d253ebb86e6bf, entries=150, sequenceid=88, filesize=11.7 K 2024-12-03T09:13:53,269 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for c49f86fc80f2af2d2b0cf03332e8cc12 in 336ms, sequenceid=88, compaction requested=false 2024-12-03T09:13:53,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:53,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:53,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-03T09:13:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-03T09:13:53,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-03T09:13:53,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 663 msec 2024-12-03T09:13:53,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 677 msec 2024-12-03T09:13:53,333 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T09:13:53,393 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T09:13:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:53,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T09:13:53,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:13:53,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:53,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:13:53,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:53,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:13:53,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:53,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/5d3e901aa1df4160805e45f369789bc5 is 50, key is test_row_0/A:col10/1733217233062/Put/seqid=0 2024-12-03T09:13:53,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217293429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217293430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217293434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217293434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217293437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741855_1031 (size=14341) 2024-12-03T09:13:53,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/5d3e901aa1df4160805e45f369789bc5 2024-12-03T09:13:53,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/91c9a49fd65042cd843c4042c49c3283 is 50, key is test_row_0/B:col10/1733217233062/Put/seqid=0 2024-12-03T09:13:53,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741856_1032 (size=12001) 2024-12-03T09:13:53,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/91c9a49fd65042cd843c4042c49c3283 2024-12-03T09:13:53,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/4747da28b42348b19d70d514f04d5d5a is 50, key is test_row_0/C:col10/1733217233062/Put/seqid=0 2024-12-03T09:13:53,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741857_1033 (size=12001) 2024-12-03T09:13:53,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/4747da28b42348b19d70d514f04d5d5a 2024-12-03T09:13:53,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/5d3e901aa1df4160805e45f369789bc5 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/5d3e901aa1df4160805e45f369789bc5 2024-12-03T09:13:53,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217293550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217293550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217293550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217293550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217293554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/5d3e901aa1df4160805e45f369789bc5, entries=200, sequenceid=109, filesize=14.0 K 2024-12-03T09:13:53,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/91c9a49fd65042cd843c4042c49c3283 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/91c9a49fd65042cd843c4042c49c3283 2024-12-03T09:13:53,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/91c9a49fd65042cd843c4042c49c3283, entries=150, sequenceid=109, filesize=11.7 K 2024-12-03T09:13:53,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/4747da28b42348b19d70d514f04d5d5a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/4747da28b42348b19d70d514f04d5d5a 2024-12-03T09:13:53,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/4747da28b42348b19d70d514f04d5d5a, entries=150, sequenceid=109, filesize=11.7 K 2024-12-03T09:13:53,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c49f86fc80f2af2d2b0cf03332e8cc12 in 185ms, sequenceid=109, compaction requested=true 2024-12-03T09:13:53,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:53,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:13:53,594 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:13:53,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:13:53,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:13:53,595 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:13:53,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:13:53,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:13:53,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:13:53,597 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:13:53,597 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/A is initiating minor compaction (all files) 2024-12-03T09:13:53,597 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:13:53,597 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/A in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:53,597 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/B is initiating minor compaction (all files) 2024-12-03T09:13:53,597 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/B in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:53,597 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/ac360c91b0584c24bcc8b011a4afd9b8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/7500b0684a234779bae58bb0ed1a16b9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/91c9a49fd65042cd843c4042c49c3283] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=35.3 K 2024-12-03T09:13:53,597 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/1fe0d613d20c4dd6b6f72c541d34d79b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/60ed4bf82f0949569cced6aac2eb48b2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/5d3e901aa1df4160805e45f369789bc5] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=37.6 K 2024-12-03T09:13:53,599 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fe0d613d20c4dd6b6f72c541d34d79b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733217231488 2024-12-03T09:13:53,599 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting ac360c91b0584c24bcc8b011a4afd9b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733217231488 2024-12-03T09:13:53,600 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7500b0684a234779bae58bb0ed1a16b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733217232693 2024-12-03T09:13:53,601 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60ed4bf82f0949569cced6aac2eb48b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733217232693 2024-12-03T09:13:53,602 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d3e901aa1df4160805e45f369789bc5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733217233062 2024-12-03T09:13:53,602 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 91c9a49fd65042cd843c4042c49c3283, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733217233062 2024-12-03T09:13:53,627 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#A#compaction#21 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:13:53,628 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/48acb7b302f64e97838bee0d8f95056c is 50, key is test_row_0/A:col10/1733217233062/Put/seqid=0 2024-12-03T09:13:53,631 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:13:53,633 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/080ea6b4695a40ae98404cc6399766fc is 50, key is test_row_0/B:col10/1733217233062/Put/seqid=0 2024-12-03T09:13:53,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741858_1034 (size=12241) 2024-12-03T09:13:53,670 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/48acb7b302f64e97838bee0d8f95056c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/48acb7b302f64e97838bee0d8f95056c 2024-12-03T09:13:53,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741859_1035 (size=12241) 2024-12-03T09:13:53,684 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/A of c49f86fc80f2af2d2b0cf03332e8cc12 into 48acb7b302f64e97838bee0d8f95056c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:13:53,685 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:53,685 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/A, priority=13, startTime=1733217233594; duration=0sec 2024-12-03T09:13:53,685 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:13:53,687 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:A 2024-12-03T09:13:53,687 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:13:53,690 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:13:53,690 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/C is initiating minor compaction (all files) 2024-12-03T09:13:53,690 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/C in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:53,691 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/ac4e9bce9a2b47508aad69922e29a0de, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/adcf3681a6ad4a0bba3d253ebb86e6bf, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/4747da28b42348b19d70d514f04d5d5a] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=35.3 K 2024-12-03T09:13:53,693 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac4e9bce9a2b47508aad69922e29a0de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1733217231488 2024-12-03T09:13:53,694 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting adcf3681a6ad4a0bba3d253ebb86e6bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733217232693 2024-12-03T09:13:53,695 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4747da28b42348b19d70d514f04d5d5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733217233062 2024-12-03T09:13:53,721 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#C#compaction#23 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:13:53,722 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/3daaaa8c3d684e1ba9ef3fe08d1d30c8 is 50, key is test_row_0/C:col10/1733217233062/Put/seqid=0 2024-12-03T09:13:53,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741860_1036 (size=12241) 2024-12-03T09:13:53,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T09:13:53,740 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:13:53,741 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:13:53,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-03T09:13:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T09:13:53,744 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:13:53,747 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:13:53,747 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:13:53,749 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/3daaaa8c3d684e1ba9ef3fe08d1d30c8 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/3daaaa8c3d684e1ba9ef3fe08d1d30c8 2024-12-03T09:13:53,764 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/C of c49f86fc80f2af2d2b0cf03332e8cc12 into 3daaaa8c3d684e1ba9ef3fe08d1d30c8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:13:53,765 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:53,765 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/C, priority=13, startTime=1733217233595; duration=0sec 2024-12-03T09:13:53,765 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:13:53,765 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:C 2024-12-03T09:13:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:53,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T09:13:53,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:13:53,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:53,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:13:53,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:53,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:13:53,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:53,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/faec359b6f0b492f8d9b3f45ab4eedbe is 50, key is test_row_0/A:col10/1733217233763/Put/seqid=0 2024-12-03T09:13:53,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741861_1037 (size=16681) 2024-12-03T09:13:53,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217293800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217293800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217293808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217293810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217293812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T09:13:53,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T09:13:53,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:53,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:53,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:53,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:53,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:53,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217293921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217293921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217293921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217293930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217293930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T09:13:54,054 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T09:13:54,058 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T09:13:54,058 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T09:13:54,059 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T09:13:54,059 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T09:13:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T09:13:54,060 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T09:13:54,060 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T09:13:54,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T09:13:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:54,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,092 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/080ea6b4695a40ae98404cc6399766fc as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/080ea6b4695a40ae98404cc6399766fc 2024-12-03T09:13:54,106 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/B of c49f86fc80f2af2d2b0cf03332e8cc12 into 080ea6b4695a40ae98404cc6399766fc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:13:54,107 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:54,107 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/B, priority=13, startTime=1733217233595; duration=0sec 2024-12-03T09:13:54,107 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:13:54,107 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:B 2024-12-03T09:13:54,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217294130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217294140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217294140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217294140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217294140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/faec359b6f0b492f8d9b3f45ab4eedbe 2024-12-03T09:13:54,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T09:13:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:54,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f63e45894a514051abf60a18abcd1ca2 is 50, key is test_row_0/B:col10/1733217233763/Put/seqid=0 2024-12-03T09:13:54,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741862_1038 (size=12001) 2024-12-03T09:13:54,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f63e45894a514051abf60a18abcd1ca2 2024-12-03T09:13:54,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/b8deec7d65c144d6abb7c707a1c397ae is 50, key is test_row_0/C:col10/1733217233763/Put/seqid=0 2024-12-03T09:13:54,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741863_1039 (size=12001) 2024-12-03T09:13:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T09:13:54,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T09:13:54,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:54,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:54,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:54,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217294439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217294450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217294450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217294450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217294451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T09:13:54,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:54,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:54,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:54,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/b8deec7d65c144d6abb7c707a1c397ae 2024-12-03T09:13:54,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T09:13:54,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:54,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:54,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:54,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:54,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/faec359b6f0b492f8d9b3f45ab4eedbe as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/faec359b6f0b492f8d9b3f45ab4eedbe 2024-12-03T09:13:54,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/faec359b6f0b492f8d9b3f45ab4eedbe, entries=250, sequenceid=127, filesize=16.3 K 2024-12-03T09:13:54,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f63e45894a514051abf60a18abcd1ca2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f63e45894a514051abf60a18abcd1ca2 2024-12-03T09:13:54,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f63e45894a514051abf60a18abcd1ca2, entries=150, sequenceid=127, filesize=11.7 K 2024-12-03T09:13:54,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/b8deec7d65c144d6abb7c707a1c397ae as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b8deec7d65c144d6abb7c707a1c397ae 2024-12-03T09:13:54,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b8deec7d65c144d6abb7c707a1c397ae, entries=150, sequenceid=127, filesize=11.7 K 2024-12-03T09:13:54,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for c49f86fc80f2af2d2b0cf03332e8cc12 in 975ms, sequenceid=127, compaction requested=false 2024-12-03T09:13:54,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:54,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T09:13:54,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:54,841 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T09:13:54,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:13:54,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:54,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:13:54,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:54,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:13:54,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:54,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/40e4620d7af445f3826b6b69acf3d356 is 50, key is test_row_0/A:col10/1733217233804/Put/seqid=0 2024-12-03T09:13:54,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741864_1040 (size=12151) 2024-12-03T09:13:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T09:13:54,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:54,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217294976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217294976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217294978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217294979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:54,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217294980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217295089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217295089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217295089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217295089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217295090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,275 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/40e4620d7af445f3826b6b69acf3d356 2024-12-03T09:13:55,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/627cf1d12725413b88a17250cba2910c is 50, key is test_row_0/B:col10/1733217233804/Put/seqid=0 2024-12-03T09:13:55,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217295300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217295301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217295301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217295301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217295301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741865_1041 (size=12151) 2024-12-03T09:13:55,312 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/627cf1d12725413b88a17250cba2910c 2024-12-03T09:13:55,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/3a6e898e546b4a049e2ee02764cac616 is 50, key is test_row_0/C:col10/1733217233804/Put/seqid=0 2024-12-03T09:13:55,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741866_1042 (size=12151) 2024-12-03T09:13:55,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217295610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217295610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217295610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217295610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217295620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:55,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:55,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:55,737 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/3a6e898e546b4a049e2ee02764cac616 2024-12-03T09:13:55,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/40e4620d7af445f3826b6b69acf3d356 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/40e4620d7af445f3826b6b69acf3d356 2024-12-03T09:13:55,758 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/40e4620d7af445f3826b6b69acf3d356, entries=150, sequenceid=145, filesize=11.9 K 2024-12-03T09:13:55,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-03T09:13:55,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/627cf1d12725413b88a17250cba2910c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/627cf1d12725413b88a17250cba2910c 2024-12-03T09:13:55,770 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/627cf1d12725413b88a17250cba2910c, entries=150, sequenceid=145, filesize=11.9 K 2024-12-03T09:13:55,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/3a6e898e546b4a049e2ee02764cac616 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/3a6e898e546b4a049e2ee02764cac616 2024-12-03T09:13:55,782 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/3a6e898e546b4a049e2ee02764cac616, entries=150, sequenceid=145, filesize=11.9 K 2024-12-03T09:13:55,783 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c49f86fc80f2af2d2b0cf03332e8cc12 in 942ms, sequenceid=145, compaction requested=true 2024-12-03T09:13:55,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:55,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:55,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-03T09:13:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-03T09:13:55,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T09:13:55,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0390 sec 2024-12-03T09:13:55,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.0550 sec 2024-12-03T09:13:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T09:13:55,890 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:13:55,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:13:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-03T09:13:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T09:13:55,893 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:13:55,894 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:13:55,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:13:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T09:13:56,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T09:13:56,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:56,049 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T09:13:56,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:13:56,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:56,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:13:56,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:56,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:13:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:56,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/e6d4952742754f1eb78c57f12230790a is 50, key is test_row_0/A:col10/1733217234976/Put/seqid=0 2024-12-03T09:13:56,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741867_1043 (size=12151) 2024-12-03T09:13:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:56,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:56,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217296134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217296142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217296143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217296144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217296148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T09:13:56,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217296249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217296259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217296260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217296260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217296260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217296460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217296469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217296469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217296469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217296469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,474 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/e6d4952742754f1eb78c57f12230790a 2024-12-03T09:13:56,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f0e8aae48b7340ba89638f24844fe83e is 50, key is test_row_0/B:col10/1733217234976/Put/seqid=0 2024-12-03T09:13:56,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741868_1044 (size=12151) 2024-12-03T09:13:56,493 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f0e8aae48b7340ba89638f24844fe83e 2024-12-03T09:13:56,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/f4b91cc0172e493680f351e30d56b3d4 is 50, key is test_row_0/C:col10/1733217234976/Put/seqid=0 2024-12-03T09:13:56,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741869_1045 (size=12151) 2024-12-03T09:13:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T09:13:56,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217296770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217296779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217296779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217296780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217296780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:56,915 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/f4b91cc0172e493680f351e30d56b3d4 2024-12-03T09:13:56,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/e6d4952742754f1eb78c57f12230790a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/e6d4952742754f1eb78c57f12230790a 2024-12-03T09:13:56,936 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/e6d4952742754f1eb78c57f12230790a, entries=150, sequenceid=160, filesize=11.9 K 2024-12-03T09:13:56,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f0e8aae48b7340ba89638f24844fe83e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f0e8aae48b7340ba89638f24844fe83e 2024-12-03T09:13:56,945 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f0e8aae48b7340ba89638f24844fe83e, entries=150, sequenceid=160, filesize=11.9 K 2024-12-03T09:13:56,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/f4b91cc0172e493680f351e30d56b3d4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/f4b91cc0172e493680f351e30d56b3d4 2024-12-03T09:13:56,957 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/f4b91cc0172e493680f351e30d56b3d4, entries=150, sequenceid=160, filesize=11.9 K 2024-12-03T09:13:56,959 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c49f86fc80f2af2d2b0cf03332e8cc12 in 909ms, sequenceid=160, compaction requested=true 2024-12-03T09:13:56,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:56,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:56,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-03T09:13:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-03T09:13:56,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T09:13:56,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0660 sec 2024-12-03T09:13:56,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.0760 sec 2024-12-03T09:13:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T09:13:57,030 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:13:57,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:13:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-03T09:13:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T09:13:57,036 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:13:57,037 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:13:57,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:13:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T09:13:57,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T09:13:57,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:57,194 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T09:13:57,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:13:57,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:57,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:13:57,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:57,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:13:57,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/b05037b67536449faa5405e6157a31dd is 50, key is test_row_0/A:col10/1733217236143/Put/seqid=0 2024-12-03T09:13:57,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741870_1046 (size=12151) 2024-12-03T09:13:57,219 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/b05037b67536449faa5405e6157a31dd 2024-12-03T09:13:57,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/5deee6304de1486fbbec0ea4e0650bb9 is 50, key is test_row_0/B:col10/1733217236143/Put/seqid=0 2024-12-03T09:13:57,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741871_1047 (size=12151) 2024-12-03T09:13:57,250 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/5deee6304de1486fbbec0ea4e0650bb9 2024-12-03T09:13:57,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/7827aa7530404718ba870beec90c900b is 50, key is test_row_0/C:col10/1733217236143/Put/seqid=0 2024-12-03T09:13:57,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:57,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741872_1048 (size=12151) 2024-12-03T09:13:57,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217297304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217297305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217297305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217297307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217297308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T09:13:57,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217297421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217297422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217297422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217297422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217297423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217297629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217297630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217297629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217297630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217297630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T09:13:57,695 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/7827aa7530404718ba870beec90c900b 2024-12-03T09:13:57,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/b05037b67536449faa5405e6157a31dd as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/b05037b67536449faa5405e6157a31dd 2024-12-03T09:13:57,711 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/b05037b67536449faa5405e6157a31dd, entries=150, sequenceid=177, filesize=11.9 K 2024-12-03T09:13:57,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/5deee6304de1486fbbec0ea4e0650bb9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5deee6304de1486fbbec0ea4e0650bb9 2024-12-03T09:13:57,719 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5deee6304de1486fbbec0ea4e0650bb9, entries=150, sequenceid=177, filesize=11.9 K 2024-12-03T09:13:57,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/7827aa7530404718ba870beec90c900b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/7827aa7530404718ba870beec90c900b 2024-12-03T09:13:57,727 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/7827aa7530404718ba870beec90c900b, entries=150, sequenceid=177, filesize=11.9 K 2024-12-03T09:13:57,728 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c49f86fc80f2af2d2b0cf03332e8cc12 in 534ms, sequenceid=177, compaction requested=true 2024-12-03T09:13:57,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:57,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:57,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-03T09:13:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-03T09:13:57,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-03T09:13:57,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 693 msec 2024-12-03T09:13:57,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 703 msec 2024-12-03T09:13:57,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:13:57,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T09:13:57,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:13:57,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:57,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:13:57,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:57,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:13:57,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:57,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217297966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217297966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217297970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:57,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217297973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217297975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:57,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/b1ff4dce61fa4131a908018331eae695 is 50, key is test_row_0/A:col10/1733217237303/Put/seqid=0 2024-12-03T09:13:58,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741873_1049 (size=12151) 2024-12-03T09:13:58,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217298079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217298079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217298090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217298090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217298090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T09:13:58,169 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:13:58,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:13:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-03T09:13:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T09:13:58,172 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:13:58,173 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:13:58,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:13:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T09:13:58,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217298289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217298289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217298300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217298300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217298300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T09:13:58,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:58,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:58,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:58,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/b1ff4dce61fa4131a908018331eae695 2024-12-03T09:13:58,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/3122bf86ffca4c70916390a8148c7db2 is 50, key is test_row_0/B:col10/1733217237303/Put/seqid=0 2024-12-03T09:13:58,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741874_1050 (size=12151) 2024-12-03T09:13:58,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T09:13:58,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:58,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:58,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:58,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T09:13:58,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217298599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217298599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217298609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217298609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217298610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:58,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:58,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T09:13:58,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:58,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:58,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:58,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T09:13:58,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:58,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:58,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:58,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T09:13:58,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/3122bf86ffca4c70916390a8148c7db2 2024-12-03T09:13:58,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/b712e0f87297498bb4b84e37c9dc0ec0 is 50, key is test_row_0/C:col10/1733217237303/Put/seqid=0 2024-12-03T09:13:58,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741875_1051 (size=12151) 2024-12-03T09:13:58,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T09:13:58,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:58,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:58,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:58,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:58,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:59,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T09:13:59,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:59,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:59,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:59,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:59,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217299110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217299110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:59,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:59,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217299119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:59,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217299119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:59,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:13:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217299119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:13:59,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:59,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:13:59,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:13:59,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T09:13:59,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:13:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:59,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:13:59,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/b712e0f87297498bb4b84e37c9dc0ec0 2024-12-03T09:13:59,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/b1ff4dce61fa4131a908018331eae695 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/b1ff4dce61fa4131a908018331eae695 2024-12-03T09:13:59,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/b1ff4dce61fa4131a908018331eae695, entries=150, sequenceid=194, filesize=11.9 K 2024-12-03T09:13:59,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/3122bf86ffca4c70916390a8148c7db2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/3122bf86ffca4c70916390a8148c7db2 2024-12-03T09:13:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T09:13:59,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/3122bf86ffca4c70916390a8148c7db2, entries=150, sequenceid=194, filesize=11.9 K 2024-12-03T09:13:59,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/b712e0f87297498bb4b84e37c9dc0ec0 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b712e0f87297498bb4b84e37c9dc0ec0 2024-12-03T09:13:59,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b712e0f87297498bb4b84e37c9dc0ec0, entries=150, sequenceid=194, filesize=11.9 K 2024-12-03T09:13:59,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c49f86fc80f2af2d2b0cf03332e8cc12 in 1377ms, sequenceid=194, compaction requested=true 2024-12-03T09:13:59,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:59,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:13:59,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:13:59,321 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-03T09:13:59,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:13:59,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:13:59,321 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-03T09:13:59,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:13:59,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:13:59,324 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72846 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-03T09:13:59,324 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 77526 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-03T09:13:59,324 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/B is initiating minor compaction (all files) 2024-12-03T09:13:59,324 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/A is initiating minor compaction (all files) 2024-12-03T09:13:59,324 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/B in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:59,324 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/A in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:59,324 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/48acb7b302f64e97838bee0d8f95056c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/faec359b6f0b492f8d9b3f45ab4eedbe, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/40e4620d7af445f3826b6b69acf3d356, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/e6d4952742754f1eb78c57f12230790a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/b05037b67536449faa5405e6157a31dd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/b1ff4dce61fa4131a908018331eae695] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=75.7 K 2024-12-03T09:13:59,324 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/080ea6b4695a40ae98404cc6399766fc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f63e45894a514051abf60a18abcd1ca2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/627cf1d12725413b88a17250cba2910c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f0e8aae48b7340ba89638f24844fe83e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5deee6304de1486fbbec0ea4e0650bb9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/3122bf86ffca4c70916390a8148c7db2] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=71.1 K 2024-12-03T09:13:59,325 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48acb7b302f64e97838bee0d8f95056c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733217233062 2024-12-03T09:13:59,325 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 080ea6b4695a40ae98404cc6399766fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733217233062 2024-12-03T09:13:59,325 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f63e45894a514051abf60a18abcd1ca2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733217233429 2024-12-03T09:13:59,325 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting faec359b6f0b492f8d9b3f45ab4eedbe, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733217233424 2024-12-03T09:13:59,326 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 627cf1d12725413b88a17250cba2910c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733217233804 2024-12-03T09:13:59,326 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40e4620d7af445f3826b6b69acf3d356, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733217233804 2024-12-03T09:13:59,326 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6d4952742754f1eb78c57f12230790a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733217234964 2024-12-03T09:13:59,326 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f0e8aae48b7340ba89638f24844fe83e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733217234964 2024-12-03T09:13:59,327 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b05037b67536449faa5405e6157a31dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733217236140 2024-12-03T09:13:59,327 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5deee6304de1486fbbec0ea4e0650bb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733217236140 2024-12-03T09:13:59,327 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1ff4dce61fa4131a908018331eae695, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733217237303 2024-12-03T09:13:59,327 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3122bf86ffca4c70916390a8148c7db2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733217237303 2024-12-03T09:13:59,352 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:13:59,353 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f9387b855f844d1f83959d1da5199fbf is 50, key is test_row_0/B:col10/1733217237303/Put/seqid=0 2024-12-03T09:13:59,359 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#A#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:13:59,360 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/c8cdefd32e9c40d998a18c3a455fb85c is 50, key is test_row_0/A:col10/1733217237303/Put/seqid=0 2024-12-03T09:13:59,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741876_1052 (size=12595) 2024-12-03T09:13:59,385 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f9387b855f844d1f83959d1da5199fbf as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f9387b855f844d1f83959d1da5199fbf 2024-12-03T09:13:59,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741877_1053 (size=12595) 2024-12-03T09:13:59,396 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/B of c49f86fc80f2af2d2b0cf03332e8cc12 into f9387b855f844d1f83959d1da5199fbf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:13:59,396 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:59,396 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/B, priority=10, startTime=1733217239321; duration=0sec 2024-12-03T09:13:59,396 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:13:59,396 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:B 2024-12-03T09:13:59,396 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-03T09:13:59,399 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72846 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-03T09:13:59,399 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/C is initiating minor compaction (all files) 2024-12-03T09:13:59,399 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/C in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:59,399 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/3daaaa8c3d684e1ba9ef3fe08d1d30c8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b8deec7d65c144d6abb7c707a1c397ae, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/3a6e898e546b4a049e2ee02764cac616, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/f4b91cc0172e493680f351e30d56b3d4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/7827aa7530404718ba870beec90c900b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b712e0f87297498bb4b84e37c9dc0ec0] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=71.1 K 2024-12-03T09:13:59,400 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3daaaa8c3d684e1ba9ef3fe08d1d30c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733217233062 2024-12-03T09:13:59,400 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting b8deec7d65c144d6abb7c707a1c397ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1733217233429 2024-12-03T09:13:59,400 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a6e898e546b4a049e2ee02764cac616, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733217233804 2024-12-03T09:13:59,401 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f4b91cc0172e493680f351e30d56b3d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1733217234964 2024-12-03T09:13:59,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T09:13:59,401 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7827aa7530404718ba870beec90c900b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733217236140 2024-12-03T09:13:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:13:59,401 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T09:13:59,402 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting b712e0f87297498bb4b84e37c9dc0ec0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733217237303 2024-12-03T09:13:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:13:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:13:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:13:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:13:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/2171a3b30f824f5887c0605764fbacba is 50, key is test_row_0/A:col10/1733217237965/Put/seqid=0 2024-12-03T09:13:59,422 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:13:59,423 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/dedc615526194f48b7f3b085e0e83618 is 50, key is test_row_0/C:col10/1733217237303/Put/seqid=0 2024-12-03T09:13:59,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741878_1054 (size=12151) 2024-12-03T09:13:59,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741879_1055 (size=12595) 2024-12-03T09:13:59,793 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/c8cdefd32e9c40d998a18c3a455fb85c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/c8cdefd32e9c40d998a18c3a455fb85c 2024-12-03T09:13:59,799 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/A of c49f86fc80f2af2d2b0cf03332e8cc12 into c8cdefd32e9c40d998a18c3a455fb85c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:13:59,799 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:59,799 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/A, priority=10, startTime=1733217239321; duration=0sec 2024-12-03T09:13:59,799 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:13:59,799 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:A 2024-12-03T09:13:59,827 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/2171a3b30f824f5887c0605764fbacba 2024-12-03T09:13:59,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/2e69360d9dec48528694bfdbc56c1cf2 is 50, key is test_row_0/B:col10/1733217237965/Put/seqid=0 2024-12-03T09:13:59,841 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/dedc615526194f48b7f3b085e0e83618 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/dedc615526194f48b7f3b085e0e83618 2024-12-03T09:13:59,849 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/C of c49f86fc80f2af2d2b0cf03332e8cc12 into dedc615526194f48b7f3b085e0e83618(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:13:59,849 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:13:59,849 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/C, priority=10, startTime=1733217239321; duration=0sec 2024-12-03T09:13:59,850 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:13:59,850 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:C 2024-12-03T09:13:59,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741880_1056 (size=12151) 2024-12-03T09:14:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:00,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:00,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217300145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217300144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217300153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217300161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217300162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217300269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217300269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217300270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217300270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217300270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,272 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/2e69360d9dec48528694bfdbc56c1cf2 2024-12-03T09:14:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/fd05d272228d4e40bb4b4b88c894cbc5 is 50, key is test_row_0/C:col10/1733217237965/Put/seqid=0 2024-12-03T09:14:00,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741881_1057 (size=12151) 2024-12-03T09:14:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T09:14:00,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217300479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217300480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217300480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217300480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217300480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,691 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/fd05d272228d4e40bb4b4b88c894cbc5 2024-12-03T09:14:00,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/2171a3b30f824f5887c0605764fbacba as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2171a3b30f824f5887c0605764fbacba 2024-12-03T09:14:00,711 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2171a3b30f824f5887c0605764fbacba, entries=150, sequenceid=210, filesize=11.9 K 2024-12-03T09:14:00,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/2e69360d9dec48528694bfdbc56c1cf2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/2e69360d9dec48528694bfdbc56c1cf2 2024-12-03T09:14:00,719 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/2e69360d9dec48528694bfdbc56c1cf2, entries=150, sequenceid=210, filesize=11.9 K 2024-12-03T09:14:00,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/fd05d272228d4e40bb4b4b88c894cbc5 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/fd05d272228d4e40bb4b4b88c894cbc5 2024-12-03T09:14:00,730 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/fd05d272228d4e40bb4b4b88c894cbc5, entries=150, sequenceid=210, filesize=11.9 K 2024-12-03T09:14:00,731 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c49f86fc80f2af2d2b0cf03332e8cc12 in 1330ms, sequenceid=210, compaction requested=false 2024-12-03T09:14:00,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:00,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:00,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-03T09:14:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-03T09:14:00,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T09:14:00,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5600 sec 2024-12-03T09:14:00,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.5680 sec 2024-12-03T09:14:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:00,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T09:14:00,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:00,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:00,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:00,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:00,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:00,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:00,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/fbad328f1109480dbcb8faeeb59b4301 is 50, key is test_row_0/A:col10/1733217240144/Put/seqid=0 2024-12-03T09:14:00,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741882_1058 (size=12151) 2024-12-03T09:14:00,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217300839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217300839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217300840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217300841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217300844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217300949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217300949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217300960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217300960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:00,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217300960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:00,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:01,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217301159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:01,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217301160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:01,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217301169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:01,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:01,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:01,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217301169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217301169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:01,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/fbad328f1109480dbcb8faeeb59b4301 2024-12-03T09:14:01,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/47078ddae9a74d5dbf46b9748a7f8404 is 50, key is test_row_0/B:col10/1733217240144/Put/seqid=0 2024-12-03T09:14:01,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741883_1059 (size=12151) 2024-12-03T09:14:01,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217301470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217301470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:01,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217301479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217301480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:01,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217301480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:01,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/47078ddae9a74d5dbf46b9748a7f8404 2024-12-03T09:14:01,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/5e5d82b4f72c4342af0f44a33a8ccee1 is 50, key is test_row_0/C:col10/1733217240144/Put/seqid=0 2024-12-03T09:14:01,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741884_1060 (size=12151) 2024-12-03T09:14:01,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/5e5d82b4f72c4342af0f44a33a8ccee1 2024-12-03T09:14:01,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/fbad328f1109480dbcb8faeeb59b4301 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/fbad328f1109480dbcb8faeeb59b4301 2024-12-03T09:14:01,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/fbad328f1109480dbcb8faeeb59b4301, entries=150, sequenceid=232, filesize=11.9 K 2024-12-03T09:14:01,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/47078ddae9a74d5dbf46b9748a7f8404 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/47078ddae9a74d5dbf46b9748a7f8404 2024-12-03T09:14:01,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/47078ddae9a74d5dbf46b9748a7f8404, entries=150, sequenceid=232, filesize=11.9 K 2024-12-03T09:14:01,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/5e5d82b4f72c4342af0f44a33a8ccee1 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/5e5d82b4f72c4342af0f44a33a8ccee1 2024-12-03T09:14:01,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/5e5d82b4f72c4342af0f44a33a8ccee1, entries=150, sequenceid=232, filesize=11.9 K 2024-12-03T09:14:01,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c49f86fc80f2af2d2b0cf03332e8cc12 in 968ms, sequenceid=232, compaction requested=true 2024-12-03T09:14:01,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:01,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:14:01,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:01,761 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:01,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:14:01,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:01,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:14:01,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:01,762 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:01,763 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:01,763 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/B is initiating minor compaction (all files) 2024-12-03T09:14:01,763 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/B in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:01,763 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f9387b855f844d1f83959d1da5199fbf, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/2e69360d9dec48528694bfdbc56c1cf2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/47078ddae9a74d5dbf46b9748a7f8404] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=36.0 K 2024-12-03T09:14:01,763 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:01,764 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/A is initiating minor compaction (all files) 2024-12-03T09:14:01,764 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/A in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:01,764 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/c8cdefd32e9c40d998a18c3a455fb85c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2171a3b30f824f5887c0605764fbacba, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/fbad328f1109480dbcb8faeeb59b4301] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=36.0 K 2024-12-03T09:14:01,765 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8cdefd32e9c40d998a18c3a455fb85c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733217237303 2024-12-03T09:14:01,765 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f9387b855f844d1f83959d1da5199fbf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733217237303 2024-12-03T09:14:01,766 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2171a3b30f824f5887c0605764fbacba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733217237965 2024-12-03T09:14:01,766 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e69360d9dec48528694bfdbc56c1cf2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733217237965 2024-12-03T09:14:01,766 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbad328f1109480dbcb8faeeb59b4301, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733217240144 2024-12-03T09:14:01,767 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 47078ddae9a74d5dbf46b9748a7f8404, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733217240144 2024-12-03T09:14:01,777 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#A#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:01,778 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/3f3813001c944a56a136c150da6cd0e4 is 50, key is test_row_0/A:col10/1733217240144/Put/seqid=0 2024-12-03T09:14:01,782 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#B#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:01,783 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/aac21e60572740f19d6e748d274469bc is 50, key is test_row_0/B:col10/1733217240144/Put/seqid=0 2024-12-03T09:14:01,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741885_1061 (size=12697) 2024-12-03T09:14:01,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741886_1062 (size=12697) 2024-12-03T09:14:01,821 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/aac21e60572740f19d6e748d274469bc as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/aac21e60572740f19d6e748d274469bc 2024-12-03T09:14:01,823 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/3f3813001c944a56a136c150da6cd0e4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/3f3813001c944a56a136c150da6cd0e4 2024-12-03T09:14:01,830 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/A of c49f86fc80f2af2d2b0cf03332e8cc12 into 3f3813001c944a56a136c150da6cd0e4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:01,831 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/B of c49f86fc80f2af2d2b0cf03332e8cc12 into aac21e60572740f19d6e748d274469bc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:01,831 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:01,831 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:01,831 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/A, priority=13, startTime=1733217241761; duration=0sec 2024-12-03T09:14:01,831 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/B, priority=13, startTime=1733217241761; duration=0sec 2024-12-03T09:14:01,831 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:01,831 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:A 2024-12-03T09:14:01,831 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:01,831 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:B 2024-12-03T09:14:01,831 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:01,832 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:01,832 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/C is initiating minor compaction (all files) 2024-12-03T09:14:01,832 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/C in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:01,832 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/dedc615526194f48b7f3b085e0e83618, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/fd05d272228d4e40bb4b4b88c894cbc5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/5e5d82b4f72c4342af0f44a33a8ccee1] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=36.0 K 2024-12-03T09:14:01,833 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting dedc615526194f48b7f3b085e0e83618, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733217237303 2024-12-03T09:14:01,834 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd05d272228d4e40bb4b4b88c894cbc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733217237965 2024-12-03T09:14:01,834 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e5d82b4f72c4342af0f44a33a8ccee1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733217240144 2024-12-03T09:14:01,850 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#C#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:01,851 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/00dd2f19d781476bb25eb8df574d4ff8 is 50, key is test_row_0/C:col10/1733217240144/Put/seqid=0 2024-12-03T09:14:01,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741887_1063 (size=12697) 2024-12-03T09:14:01,872 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/00dd2f19d781476bb25eb8df574d4ff8 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/00dd2f19d781476bb25eb8df574d4ff8 2024-12-03T09:14:01,881 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/C of c49f86fc80f2af2d2b0cf03332e8cc12 into 00dd2f19d781476bb25eb8df574d4ff8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:01,881 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:01,881 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/C, priority=13, startTime=1733217241762; duration=0sec 2024-12-03T09:14:01,881 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:01,882 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:C 2024-12-03T09:14:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:01,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T09:14:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:02,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/041625aae23a4319adc4cb9b77604bc2 is 50, key is test_row_0/A:col10/1733217241980/Put/seqid=0 2024-12-03T09:14:02,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741888_1064 (size=12151) 2024-12-03T09:14:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217302016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217302016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/041625aae23a4319adc4cb9b77604bc2 2024-12-03T09:14:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217302018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217302018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217302019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/5d15cd599d8c4593bffe00d5dfb229ef is 50, key is test_row_0/B:col10/1733217241980/Put/seqid=0 2024-12-03T09:14:02,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741889_1065 (size=12151) 2024-12-03T09:14:02,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217302130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217302130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217302130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217302130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217302131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T09:14:02,329 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:02,330 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-03T09:14:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T09:14:02,333 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:02,333 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:02,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:02,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217302340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217302340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217302340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217302342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217302341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T09:14:02,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/5d15cd599d8c4593bffe00d5dfb229ef 2024-12-03T09:14:02,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/6406c36a5a234899bcd1a699d91673c5 is 50, key is test_row_0/C:col10/1733217241980/Put/seqid=0 2024-12-03T09:14:02,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741890_1066 (size=12151) 2024-12-03T09:14:02,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/6406c36a5a234899bcd1a699d91673c5 2024-12-03T09:14:02,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/041625aae23a4319adc4cb9b77604bc2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/041625aae23a4319adc4cb9b77604bc2 2024-12-03T09:14:02,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/041625aae23a4319adc4cb9b77604bc2, entries=150, sequenceid=248, filesize=11.9 K 2024-12-03T09:14:02,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T09:14:02,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:02,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:02,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:02,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:02,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:02,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:02,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/5d15cd599d8c4593bffe00d5dfb229ef as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5d15cd599d8c4593bffe00d5dfb229ef 2024-12-03T09:14:02,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5d15cd599d8c4593bffe00d5dfb229ef, entries=150, sequenceid=248, filesize=11.9 K 2024-12-03T09:14:02,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/6406c36a5a234899bcd1a699d91673c5 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/6406c36a5a234899bcd1a699d91673c5 2024-12-03T09:14:02,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/6406c36a5a234899bcd1a699d91673c5, entries=150, sequenceid=248, filesize=11.9 K 2024-12-03T09:14:02,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c49f86fc80f2af2d2b0cf03332e8cc12 in 532ms, sequenceid=248, compaction requested=false 2024-12-03T09:14:02,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:02,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T09:14:02,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:02,645 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T09:14:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:02,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T09:14:02,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:02,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:02,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/39e18fadd94d4c3c8215634732a19c21 is 50, key is test_row_0/A:col10/1733217242017/Put/seqid=0 2024-12-03T09:14:02,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741891_1067 (size=12301) 2024-12-03T09:14:02,658 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/39e18fadd94d4c3c8215634732a19c21 2024-12-03T09:14:02,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217302659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217302659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217302663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217302665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217302665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/4232e4b93b544949ac36de04c8996884 is 50, key is test_row_0/B:col10/1733217242017/Put/seqid=0 2024-12-03T09:14:02,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741892_1068 (size=12301) 2024-12-03T09:14:02,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217302770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217302770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217302779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217302779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217302780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T09:14:02,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217302979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217302980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217302990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217302990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217302990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,092 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/4232e4b93b544949ac36de04c8996884 2024-12-03T09:14:03,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/456a13fc1a344294bf2a8ce62e6177df is 50, key is test_row_0/C:col10/1733217242017/Put/seqid=0 2024-12-03T09:14:03,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741893_1069 (size=12301) 2024-12-03T09:14:03,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217303289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217303289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217303299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217303299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217303299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T09:14:03,515 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/456a13fc1a344294bf2a8ce62e6177df 2024-12-03T09:14:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/39e18fadd94d4c3c8215634732a19c21 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/39e18fadd94d4c3c8215634732a19c21 2024-12-03T09:14:03,535 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/39e18fadd94d4c3c8215634732a19c21, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T09:14:03,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/4232e4b93b544949ac36de04c8996884 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/4232e4b93b544949ac36de04c8996884 2024-12-03T09:14:03,546 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/4232e4b93b544949ac36de04c8996884, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T09:14:03,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/456a13fc1a344294bf2a8ce62e6177df as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/456a13fc1a344294bf2a8ce62e6177df 2024-12-03T09:14:03,558 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/456a13fc1a344294bf2a8ce62e6177df, entries=150, sequenceid=267, filesize=12.0 K 2024-12-03T09:14:03,560 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c49f86fc80f2af2d2b0cf03332e8cc12 in 914ms, sequenceid=267, compaction requested=true 2024-12-03T09:14:03,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:03,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:03,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-03T09:14:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-03T09:14:03,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T09:14:03,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2280 sec 2024-12-03T09:14:03,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.2380 sec 2024-12-03T09:14:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:03,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T09:14:03,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:03,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:03,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:03,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:03,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:03,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:03,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/8103aaa2fd914ed8a6d865f1a36db67d is 50, key is test_row_0/A:col10/1733217243799/Put/seqid=0 2024-12-03T09:14:03,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741894_1070 (size=14741) 2024-12-03T09:14:03,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217303830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217303829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/8103aaa2fd914ed8a6d865f1a36db67d 2024-12-03T09:14:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217303837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217303838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217303838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/a5be33be05ad41c191d781364cdd616f is 50, key is test_row_0/B:col10/1733217243799/Put/seqid=0 2024-12-03T09:14:03,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741895_1071 (size=12301) 2024-12-03T09:14:03,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217303949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217303949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217303950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217303950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:03,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217303950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217304160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217304160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217304160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217304160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217304160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/a5be33be05ad41c191d781364cdd616f 2024-12-03T09:14:04,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/d0f8bc9c916047439464f51526ef9e8f is 50, key is test_row_0/C:col10/1733217243799/Put/seqid=0 2024-12-03T09:14:04,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741896_1072 (size=12301) 2024-12-03T09:14:04,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217304470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217304470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217304470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217304470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217304470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T09:14:04,480 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:04,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-03T09:14:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T09:14:04,484 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:04,485 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:04,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T09:14:04,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T09:14:04,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:04,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:04,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:04,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:04,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:04,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/d0f8bc9c916047439464f51526ef9e8f 2024-12-03T09:14:04,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/8103aaa2fd914ed8a6d865f1a36db67d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/8103aaa2fd914ed8a6d865f1a36db67d 2024-12-03T09:14:04,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/8103aaa2fd914ed8a6d865f1a36db67d, entries=200, sequenceid=281, filesize=14.4 K 2024-12-03T09:14:04,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/a5be33be05ad41c191d781364cdd616f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/a5be33be05ad41c191d781364cdd616f 2024-12-03T09:14:04,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/a5be33be05ad41c191d781364cdd616f, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T09:14:04,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/d0f8bc9c916047439464f51526ef9e8f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/d0f8bc9c916047439464f51526ef9e8f 2024-12-03T09:14:04,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/d0f8bc9c916047439464f51526ef9e8f, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T09:14:04,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for c49f86fc80f2af2d2b0cf03332e8cc12 in 913ms, sequenceid=281, compaction requested=true 2024-12-03T09:14:04,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:04,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:14:04,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:04,714 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:04,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:14:04,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:04,715 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:04,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:14:04,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:04,716 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51890 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:04,716 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:04,716 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/B is initiating minor compaction (all files) 2024-12-03T09:14:04,716 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/A is initiating minor compaction (all files) 2024-12-03T09:14:04,716 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/B in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:04,716 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/A in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:04,716 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/aac21e60572740f19d6e748d274469bc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5d15cd599d8c4593bffe00d5dfb229ef, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/4232e4b93b544949ac36de04c8996884, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/a5be33be05ad41c191d781364cdd616f] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=48.3 K 2024-12-03T09:14:04,716 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/3f3813001c944a56a136c150da6cd0e4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/041625aae23a4319adc4cb9b77604bc2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/39e18fadd94d4c3c8215634732a19c21, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/8103aaa2fd914ed8a6d865f1a36db67d] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=50.7 K 2024-12-03T09:14:04,716 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f3813001c944a56a136c150da6cd0e4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733217240144 2024-12-03T09:14:04,716 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting aac21e60572740f19d6e748d274469bc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733217240144 2024-12-03T09:14:04,717 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d15cd599d8c4593bffe00d5dfb229ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733217240798 2024-12-03T09:14:04,717 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 041625aae23a4319adc4cb9b77604bc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733217240798 2024-12-03T09:14:04,717 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4232e4b93b544949ac36de04c8996884, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733217242013 2024-12-03T09:14:04,717 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39e18fadd94d4c3c8215634732a19c21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733217242013 2024-12-03T09:14:04,717 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting a5be33be05ad41c191d781364cdd616f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733217242656 2024-12-03T09:14:04,717 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8103aaa2fd914ed8a6d865f1a36db67d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733217242656 2024-12-03T09:14:04,728 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:04,729 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/6a47557d246b45ef8a36918556d93df5 is 50, key is test_row_0/B:col10/1733217243799/Put/seqid=0 2024-12-03T09:14:04,731 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:04,732 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/83cdfa0540ce402b8ccdfe31a21ef986 is 50, key is test_row_0/A:col10/1733217243799/Put/seqid=0 2024-12-03T09:14:04,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741897_1073 (size=12983) 2024-12-03T09:14:04,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741898_1074 (size=12983) 2024-12-03T09:14:04,758 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/6a47557d246b45ef8a36918556d93df5 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/6a47557d246b45ef8a36918556d93df5 2024-12-03T09:14:04,766 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/B of c49f86fc80f2af2d2b0cf03332e8cc12 into 6a47557d246b45ef8a36918556d93df5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:04,766 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:04,767 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/B, priority=12, startTime=1733217244714; duration=0sec 2024-12-03T09:14:04,767 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:04,767 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:B 2024-12-03T09:14:04,767 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:04,768 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:04,768 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/C is initiating minor compaction (all files) 2024-12-03T09:14:04,768 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/C in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:04,768 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/00dd2f19d781476bb25eb8df574d4ff8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/6406c36a5a234899bcd1a699d91673c5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/456a13fc1a344294bf2a8ce62e6177df, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/d0f8bc9c916047439464f51526ef9e8f] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=48.3 K 2024-12-03T09:14:04,768 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 00dd2f19d781476bb25eb8df574d4ff8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1733217240144 2024-12-03T09:14:04,769 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6406c36a5a234899bcd1a699d91673c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733217240798 2024-12-03T09:14:04,769 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 456a13fc1a344294bf2a8ce62e6177df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733217242013 2024-12-03T09:14:04,769 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting d0f8bc9c916047439464f51526ef9e8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733217242656 2024-12-03T09:14:04,779 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:04,779 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/e8e24b1f522d4086bd4c2a5a8f6d4902 is 50, key is test_row_0/C:col10/1733217243799/Put/seqid=0 2024-12-03T09:14:04,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741899_1075 (size=12983) 2024-12-03T09:14:04,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-03T09:14:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:04,791 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T09:14:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T09:14:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/895c64172b97431c9d52029bc171be0d is 50, key is test_row_0/A:col10/1733217243829/Put/seqid=0 2024-12-03T09:14:04,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741900_1076 (size=12301) 2024-12-03T09:14:04,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:04,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:04,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217304986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217304987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217304988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217304994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217304995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217305099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217305099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217305099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T09:14:05,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217305111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217305111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,163 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/83cdfa0540ce402b8ccdfe31a21ef986 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/83cdfa0540ce402b8ccdfe31a21ef986 2024-12-03T09:14:05,180 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/A of c49f86fc80f2af2d2b0cf03332e8cc12 into 83cdfa0540ce402b8ccdfe31a21ef986(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:05,180 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:05,180 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/A, priority=12, startTime=1733217244714; duration=0sec 2024-12-03T09:14:05,180 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:05,181 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:A 2024-12-03T09:14:05,200 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/e8e24b1f522d4086bd4c2a5a8f6d4902 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/e8e24b1f522d4086bd4c2a5a8f6d4902 2024-12-03T09:14:05,209 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/895c64172b97431c9d52029bc171be0d 2024-12-03T09:14:05,211 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/C of c49f86fc80f2af2d2b0cf03332e8cc12 into e8e24b1f522d4086bd4c2a5a8f6d4902(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:05,211 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:05,211 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/C, priority=12, startTime=1733217244715; duration=0sec 2024-12-03T09:14:05,211 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:05,211 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:C 2024-12-03T09:14:05,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/0a4ae41b75a144aa9a1d607f1cf3c01d is 50, key is test_row_0/B:col10/1733217243829/Put/seqid=0 2024-12-03T09:14:05,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741901_1077 (size=12301) 2024-12-03T09:14:05,245 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/0a4ae41b75a144aa9a1d607f1cf3c01d 2024-12-03T09:14:05,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/8ede13d4b1e943109278d5d071f05b21 is 50, key is test_row_0/C:col10/1733217243829/Put/seqid=0 2024-12-03T09:14:05,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741902_1078 (size=12301) 2024-12-03T09:14:05,270 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/8ede13d4b1e943109278d5d071f05b21 2024-12-03T09:14:05,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/895c64172b97431c9d52029bc171be0d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/895c64172b97431c9d52029bc171be0d 2024-12-03T09:14:05,291 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/895c64172b97431c9d52029bc171be0d, entries=150, sequenceid=302, filesize=12.0 K 2024-12-03T09:14:05,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/0a4ae41b75a144aa9a1d607f1cf3c01d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/0a4ae41b75a144aa9a1d607f1cf3c01d 2024-12-03T09:14:05,300 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/0a4ae41b75a144aa9a1d607f1cf3c01d, entries=150, sequenceid=302, filesize=12.0 K 2024-12-03T09:14:05,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/8ede13d4b1e943109278d5d071f05b21 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/8ede13d4b1e943109278d5d071f05b21 2024-12-03T09:14:05,307 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/8ede13d4b1e943109278d5d071f05b21, entries=150, sequenceid=302, filesize=12.0 K 2024-12-03T09:14:05,309 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c49f86fc80f2af2d2b0cf03332e8cc12 in 518ms, sequenceid=302, compaction requested=false 2024-12-03T09:14:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:05,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:05,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-03T09:14:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:05,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T09:14:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-03T09:14:05,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:05,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:05,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:05,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:05,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:05,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:05,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-03T09:14:05,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 828 msec 2024-12-03T09:14:05,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 838 msec 2024-12-03T09:14:05,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/25886ac9e69f4204a66578d61c18b121 is 50, key is test_row_0/A:col10/1733217244992/Put/seqid=0 2024-12-03T09:14:05,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217305337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741903_1079 (size=14741) 2024-12-03T09:14:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217305338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217305340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/25886ac9e69f4204a66578d61c18b121 2024-12-03T09:14:05,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217305348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217305354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/b0922ee532764cf19c7f9fcc0f4b4132 is 50, key is test_row_0/B:col10/1733217244992/Put/seqid=0 2024-12-03T09:14:05,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741904_1080 (size=12301) 2024-12-03T09:14:05,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/b0922ee532764cf19c7f9fcc0f4b4132 2024-12-03T09:14:05,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/77aa87052d634da2844e8a551d872b4a is 50, key is test_row_0/C:col10/1733217244992/Put/seqid=0 2024-12-03T09:14:05,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741905_1081 (size=12301) 2024-12-03T09:14:05,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/77aa87052d634da2844e8a551d872b4a 2024-12-03T09:14:05,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/25886ac9e69f4204a66578d61c18b121 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/25886ac9e69f4204a66578d61c18b121 2024-12-03T09:14:05,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/25886ac9e69f4204a66578d61c18b121, entries=200, sequenceid=317, filesize=14.4 K 2024-12-03T09:14:05,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/b0922ee532764cf19c7f9fcc0f4b4132 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/b0922ee532764cf19c7f9fcc0f4b4132 2024-12-03T09:14:05,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/b0922ee532764cf19c7f9fcc0f4b4132, entries=150, sequenceid=317, filesize=12.0 K 2024-12-03T09:14:05,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/77aa87052d634da2844e8a551d872b4a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/77aa87052d634da2844e8a551d872b4a 2024-12-03T09:14:05,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/77aa87052d634da2844e8a551d872b4a, entries=150, sequenceid=317, filesize=12.0 K 2024-12-03T09:14:05,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c49f86fc80f2af2d2b0cf03332e8cc12 in 121ms, sequenceid=317, compaction requested=true 2024-12-03T09:14:05,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:05,432 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:05,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:14:05,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:05,433 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:05,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:14:05,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:05,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:14:05,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:05,436 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:05,436 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/A is initiating minor compaction (all files) 2024-12-03T09:14:05,436 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:05,436 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/A in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:05,437 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/B is initiating minor compaction (all files) 2024-12-03T09:14:05,437 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/B in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:05,437 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/83cdfa0540ce402b8ccdfe31a21ef986, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/895c64172b97431c9d52029bc171be0d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/25886ac9e69f4204a66578d61c18b121] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=39.1 K 2024-12-03T09:14:05,437 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/6a47557d246b45ef8a36918556d93df5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/0a4ae41b75a144aa9a1d607f1cf3c01d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/b0922ee532764cf19c7f9fcc0f4b4132] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=36.7 K 2024-12-03T09:14:05,437 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83cdfa0540ce402b8ccdfe31a21ef986, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733217242656 2024-12-03T09:14:05,437 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a47557d246b45ef8a36918556d93df5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733217242656 2024-12-03T09:14:05,437 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a4ae41b75a144aa9a1d607f1cf3c01d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733217243829 2024-12-03T09:14:05,438 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 895c64172b97431c9d52029bc171be0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733217243829 2024-12-03T09:14:05,438 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting b0922ee532764cf19c7f9fcc0f4b4132, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217244984 2024-12-03T09:14:05,438 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25886ac9e69f4204a66578d61c18b121, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217244984 2024-12-03T09:14:05,448 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:05,449 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/87653ba887cb48c5812fb00a8641c0e0 is 50, key is test_row_0/A:col10/1733217244992/Put/seqid=0 2024-12-03T09:14:05,454 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#B#compaction#70 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:05,454 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f364ae6e9f7e4b9bb89df652b916ec7e is 50, key is test_row_0/B:col10/1733217244992/Put/seqid=0 2024-12-03T09:14:05,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217245328, max timestamp=1733217245452Num uniques 859; 2024-12-03T09:14:05,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:05,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217245328, max timestamp=1733217245452Num uniques 859; 2024-12-03T09:14:05,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:05,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-12-03T09:14:05,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741907_1083 (size=13085) 2024-12-03T09:14:05,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:05,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:05,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217245328, max timestamp=1733217245452Num uniques 859; 2024-12-03T09:14:05,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:05,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:05,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:05,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:05,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:05,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741906_1082 (size=13085) 2024-12-03T09:14:05,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/ed6361c740f545f99fd53b6f2a6017a1 is 50, key is test_row_0/A:col10/1733217245452/Put/seqid=0 2024-12-03T09:14:05,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741908_1084 (size=17181) 2024-12-03T09:14:05,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/ed6361c740f545f99fd53b6f2a6017a1 2024-12-03T09:14:05,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/931983e52ccf45e894a9e15c9c40fd39 is 50, key is test_row_0/B:col10/1733217245452/Put/seqid=0 2024-12-03T09:14:05,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741909_1085 (size=12301) 2024-12-03T09:14:05,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217245496, max timestamp=1733217245510Num uniques 859; 2024-12-03T09:14:05,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:05,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217245496, max timestamp=1733217245510Num uniques 859; 2024-12-03T09:14:05,521 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:05,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217245496, max timestamp=1733217245510Num uniques 859; 2024-12-03T09:14:05,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:05,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733217245540, max timestamp=1733217245550Num uniques 1018; 2024-12-03T09:14:05,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733217245540, max timestamp=1733217245550Num uniques 1018; 2024-12-03T09:14:05,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T09:14:05,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T09:14:05,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733217245540, max timestamp=1733217245550Num uniques 1018; 2024-12-03T09:14:05,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T09:14:05,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217305589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217305593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=477, cellSize=21842, totalHeapSize=49848, min timestamp=1733217245568, max timestamp=1733217245593Num uniques 1127; 2024-12-03T09:14:05,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=21842, suffix heap size=145176,new segment heap size=49848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=477 2024-12-03T09:14:05,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=477, cellSize=21842, totalHeapSize=49848, min timestamp=1733217245568, max timestamp=1733217245593Num uniques 1127; 2024-12-03T09:14:05,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=21842, suffix heap size=145176,new segment heap size=49848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=477 2024-12-03T09:14:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733217245587, max timestamp=1733217245595Num uniques 1127; 2024-12-03T09:14:05,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-03T09:14:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T09:14:05,620 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:05,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-03T09:14:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T09:14:05,624 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:05,626 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:05,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:05,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217305634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217305634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217305636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217305709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217305709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T09:14:05,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217305749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217305749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217305749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T09:14:05,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:05,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:05,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:05,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:05,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:05,872 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f364ae6e9f7e4b9bb89df652b916ec7e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f364ae6e9f7e4b9bb89df652b916ec7e 2024-12-03T09:14:05,876 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/87653ba887cb48c5812fb00a8641c0e0 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/87653ba887cb48c5812fb00a8641c0e0 2024-12-03T09:14:05,878 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/B of c49f86fc80f2af2d2b0cf03332e8cc12 into f364ae6e9f7e4b9bb89df652b916ec7e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:05,878 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:05,879 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/B, priority=13, startTime=1733217245433; duration=0sec 2024-12-03T09:14:05,879 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:05,879 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:B 2024-12-03T09:14:05,879 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:05,881 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:05,881 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/C is initiating minor compaction (all files) 2024-12-03T09:14:05,881 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/C in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:05,881 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/e8e24b1f522d4086bd4c2a5a8f6d4902, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/8ede13d4b1e943109278d5d071f05b21, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/77aa87052d634da2844e8a551d872b4a] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=36.7 K 2024-12-03T09:14:05,885 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting e8e24b1f522d4086bd4c2a5a8f6d4902, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733217242656 2024-12-03T09:14:05,886 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ede13d4b1e943109278d5d071f05b21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733217243829 2024-12-03T09:14:05,886 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/A of c49f86fc80f2af2d2b0cf03332e8cc12 into 87653ba887cb48c5812fb00a8641c0e0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:05,886 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:05,886 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/A, priority=13, startTime=1733217245432; duration=0sec 2024-12-03T09:14:05,886 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:05,886 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 77aa87052d634da2844e8a551d872b4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217244984 2024-12-03T09:14:05,886 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:A 2024-12-03T09:14:05,895 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#C#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:05,896 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/f83a591484f44d209678b097d8e6ebdd is 50, key is test_row_0/C:col10/1733217244992/Put/seqid=0 2024-12-03T09:14:05,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741910_1086 (size=13085) 2024-12-03T09:14:05,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/931983e52ccf45e894a9e15c9c40fd39 2024-12-03T09:14:05,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217305920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217305920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/0dcd854fd8c04b2192d4b4c75a825c70 is 50, key is test_row_0/C:col10/1733217245452/Put/seqid=0 2024-12-03T09:14:05,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T09:14:05,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:05,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:05,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:05,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:05,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T09:14:05,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741911_1087 (size=12301) 2024-12-03T09:14:05,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/0dcd854fd8c04b2192d4b4c75a825c70 2024-12-03T09:14:05,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/ed6361c740f545f99fd53b6f2a6017a1 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/ed6361c740f545f99fd53b6f2a6017a1 2024-12-03T09:14:05,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/ed6361c740f545f99fd53b6f2a6017a1, entries=250, sequenceid=339, filesize=16.8 K 2024-12-03T09:14:05,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/931983e52ccf45e894a9e15c9c40fd39 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/931983e52ccf45e894a9e15c9c40fd39 2024-12-03T09:14:05,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217305960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217305964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217305964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:05,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/931983e52ccf45e894a9e15c9c40fd39, entries=150, sequenceid=339, filesize=12.0 K 2024-12-03T09:14:05,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/0dcd854fd8c04b2192d4b4c75a825c70 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/0dcd854fd8c04b2192d4b4c75a825c70 2024-12-03T09:14:05,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/0dcd854fd8c04b2192d4b4c75a825c70, entries=150, sequenceid=339, filesize=12.0 K 2024-12-03T09:14:05,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=161.02 KB/164880 for c49f86fc80f2af2d2b0cf03332e8cc12 in 519ms, sequenceid=339, compaction requested=false 2024-12-03T09:14:05,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:06,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-03T09:14:06,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:06,086 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=161.02 KB heapSize=444.23 KB 2024-12-03T09:14:06,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:06,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:06,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:06,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:06,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:06,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:06,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/689e5336b1ca438ca1ca6b1f1812b49b is 50, key is test_row_0/A:col10/1733217245632/Put/seqid=0 2024-12-03T09:14:06,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741912_1088 (size=12301) 2024-12-03T09:14:06,108 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/689e5336b1ca438ca1ca6b1f1812b49b 2024-12-03T09:14:06,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f42b79c5c6e649c09641e93845039832 is 50, key is test_row_0/B:col10/1733217245632/Put/seqid=0 2024-12-03T09:14:06,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741913_1089 (size=12301) 2024-12-03T09:14:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:06,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:06,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217306234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217306241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T09:14:06,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217306279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217306279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217306280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,318 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/f83a591484f44d209678b097d8e6ebdd as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/f83a591484f44d209678b097d8e6ebdd 2024-12-03T09:14:06,326 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/C of c49f86fc80f2af2d2b0cf03332e8cc12 into f83a591484f44d209678b097d8e6ebdd(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:06,326 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:06,326 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/C, priority=13, startTime=1733217245433; duration=0sec 2024-12-03T09:14:06,326 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:06,326 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:C 2024-12-03T09:14:06,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217306349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217306350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,540 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f42b79c5c6e649c09641e93845039832 2024-12-03T09:14:06,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/a26d235e06be44848d97af2120918a53 is 50, key is test_row_0/C:col10/1733217245632/Put/seqid=0 2024-12-03T09:14:06,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741914_1090 (size=12301) 2024-12-03T09:14:06,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217306559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217306559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T09:14:06,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217306789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217306789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217306789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217306869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217306869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:06,953 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/a26d235e06be44848d97af2120918a53 2024-12-03T09:14:06,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/689e5336b1ca438ca1ca6b1f1812b49b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/689e5336b1ca438ca1ca6b1f1812b49b 2024-12-03T09:14:06,964 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/689e5336b1ca438ca1ca6b1f1812b49b, entries=150, sequenceid=412, filesize=12.0 K 2024-12-03T09:14:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f42b79c5c6e649c09641e93845039832 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f42b79c5c6e649c09641e93845039832 2024-12-03T09:14:06,972 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f42b79c5c6e649c09641e93845039832, entries=150, sequenceid=412, filesize=12.0 K 2024-12-03T09:14:06,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/a26d235e06be44848d97af2120918a53 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/a26d235e06be44848d97af2120918a53 2024-12-03T09:14:06,982 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/a26d235e06be44848d97af2120918a53, entries=150, sequenceid=412, filesize=12.0 K 2024-12-03T09:14:06,983 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~444.19 KB/454848, currentSize=26.84 KB/27480 for c49f86fc80f2af2d2b0cf03332e8cc12 in 897ms, sequenceid=412, compaction requested=true 2024-12-03T09:14:06,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:06,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:06,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-03T09:14:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-03T09:14:06,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T09:14:06,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3600 sec 2024-12-03T09:14:06,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.3720 sec 2024-12-03T09:14:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:07,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:14:07,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:07,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:07,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:07,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:07,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:07,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:07,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/2b121a5637a1498eb30f77b021b780bc is 50, key is test_row_0/A:col10/1733217246239/Put/seqid=0 2024-12-03T09:14:07,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741915_1091 (size=17181) 2024-12-03T09:14:07,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/2b121a5637a1498eb30f77b021b780bc 2024-12-03T09:14:07,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/579c7a79c66946219951b3738c0c13e3 is 50, key is test_row_0/B:col10/1733217246239/Put/seqid=0 2024-12-03T09:14:07,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741916_1092 (size=12301) 2024-12-03T09:14:07,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217247393, max timestamp=1733217247435Num uniques 859; 2024-12-03T09:14:07,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217247393, max timestamp=1733217247435Num uniques 859; 2024-12-03T09:14:07,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:07,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:07,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217247393, max timestamp=1733217247435Num uniques 859; 2024-12-03T09:14:07,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:07,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217307479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:07,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217307480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:07,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217307590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217307590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:07,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-03T09:14:07,770 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:07,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-03T09:14:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T09:14:07,772 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:07,773 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:07,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:07,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:07,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217307800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:07,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217307800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:07,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217307800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:07,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217307800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:07,810 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in c49f86fc80f2af2d2b0cf03332e8cc12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:07,810 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in c49f86fc80f2af2d2b0cf03332e8cc12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:07,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:07,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:07,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:07,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217307809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:07,814 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in c49f86fc80f2af2d2b0cf03332e8cc12 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:07,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/579c7a79c66946219951b3738c0c13e3 2024-12-03T09:14:07,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/46e2060b9e0b4e53a634123d8277cbdf is 50, key is test_row_0/C:col10/1733217246239/Put/seqid=0 2024-12-03T09:14:07,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741917_1093 (size=12301) 2024-12-03T09:14:07,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/46e2060b9e0b4e53a634123d8277cbdf 2024-12-03T09:14:07,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/2b121a5637a1498eb30f77b021b780bc as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2b121a5637a1498eb30f77b021b780bc 2024-12-03T09:14:07,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2b121a5637a1498eb30f77b021b780bc, entries=250, sequenceid=423, filesize=16.8 K 2024-12-03T09:14:07,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/579c7a79c66946219951b3738c0c13e3 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/579c7a79c66946219951b3738c0c13e3 2024-12-03T09:14:07,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/579c7a79c66946219951b3738c0c13e3, entries=150, sequenceid=423, filesize=12.0 K 2024-12-03T09:14:07,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/46e2060b9e0b4e53a634123d8277cbdf as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/46e2060b9e0b4e53a634123d8277cbdf 2024-12-03T09:14:07,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/46e2060b9e0b4e53a634123d8277cbdf, entries=150, sequenceid=423, filesize=12.0 K 2024-12-03T09:14:07,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c49f86fc80f2af2d2b0cf03332e8cc12 in 483ms, sequenceid=423, compaction requested=true 2024-12-03T09:14:07,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:07,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:14:07,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:07,875 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:07,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:14:07,875 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:07,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:07,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:14:07,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:07,876 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:07,876 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59748 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:07,877 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/B is initiating minor compaction (all files) 2024-12-03T09:14:07,877 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/A is initiating minor compaction (all files) 2024-12-03T09:14:07,877 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/A in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:07,877 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/B in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:07,877 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/87653ba887cb48c5812fb00a8641c0e0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/ed6361c740f545f99fd53b6f2a6017a1, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/689e5336b1ca438ca1ca6b1f1812b49b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2b121a5637a1498eb30f77b021b780bc] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=58.3 K 2024-12-03T09:14:07,877 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f364ae6e9f7e4b9bb89df652b916ec7e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/931983e52ccf45e894a9e15c9c40fd39, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f42b79c5c6e649c09641e93845039832, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/579c7a79c66946219951b3738c0c13e3] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=48.8 K 2024-12-03T09:14:07,877 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f364ae6e9f7e4b9bb89df652b916ec7e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217244984 2024-12-03T09:14:07,877 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87653ba887cb48c5812fb00a8641c0e0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217244984 2024-12-03T09:14:07,878 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed6361c740f545f99fd53b6f2a6017a1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733217245328 2024-12-03T09:14:07,878 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 931983e52ccf45e894a9e15c9c40fd39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733217245345 2024-12-03T09:14:07,878 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 689e5336b1ca438ca1ca6b1f1812b49b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733217245631 2024-12-03T09:14:07,878 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f42b79c5c6e649c09641e93845039832, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733217245631 2024-12-03T09:14:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T09:14:07,879 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 579c7a79c66946219951b3738c0c13e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733217246239 2024-12-03T09:14:07,879 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b121a5637a1498eb30f77b021b780bc, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733217246233 2024-12-03T09:14:07,890 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:07,891 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:07,891 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/5df7792775f94dbe8b3122b5b1f38f99 is 50, key is test_row_0/B:col10/1733217246239/Put/seqid=0 2024-12-03T09:14:07,891 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/e328c664b5b942039b99511e09fae7ca is 50, key is test_row_0/A:col10/1733217246239/Put/seqid=0 2024-12-03T09:14:07,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741919_1095 (size=13221) 2024-12-03T09:14:07,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741918_1094 (size=13221) 2024-12-03T09:14:07,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-03T09:14:07,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:07,926 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T09:14:07,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:07,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:07,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:07,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:07,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:07,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:07,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/d44856af31b6457fbec7a019551c4f4c is 50, key is test_row_0/A:col10/1733217247474/Put/seqid=0 2024-12-03T09:14:07,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741920_1096 (size=12301) 2024-12-03T09:14:07,936 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/d44856af31b6457fbec7a019551c4f4c 2024-12-03T09:14:07,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/807aa22b85eb4eb29875e637a38c7723 is 50, key is test_row_0/B:col10/1733217247474/Put/seqid=0 2024-12-03T09:14:07,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741921_1097 (size=12301) 2024-12-03T09:14:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T09:14:08,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:08,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:08,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217308137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:08,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217308138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:08,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217308249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:08,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217308249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:08,310 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/e328c664b5b942039b99511e09fae7ca as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/e328c664b5b942039b99511e09fae7ca 2024-12-03T09:14:08,313 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/5df7792775f94dbe8b3122b5b1f38f99 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5df7792775f94dbe8b3122b5b1f38f99 2024-12-03T09:14:08,323 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/A of c49f86fc80f2af2d2b0cf03332e8cc12 into e328c664b5b942039b99511e09fae7ca(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:08,323 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:08,323 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/B of c49f86fc80f2af2d2b0cf03332e8cc12 into 5df7792775f94dbe8b3122b5b1f38f99(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:08,323 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/A, priority=12, startTime=1733217247875; duration=0sec 2024-12-03T09:14:08,323 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:08,323 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/B, priority=12, startTime=1733217247875; duration=0sec 2024-12-03T09:14:08,323 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:08,323 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:A 2024-12-03T09:14:08,323 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:08,323 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:B 2024-12-03T09:14:08,323 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:08,326 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:08,326 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/C is initiating minor compaction (all files) 2024-12-03T09:14:08,326 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/C in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:08,326 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/f83a591484f44d209678b097d8e6ebdd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/0dcd854fd8c04b2192d4b4c75a825c70, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/a26d235e06be44848d97af2120918a53, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/46e2060b9e0b4e53a634123d8277cbdf] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=48.8 K 2024-12-03T09:14:08,327 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f83a591484f44d209678b097d8e6ebdd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217244984 2024-12-03T09:14:08,327 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dcd854fd8c04b2192d4b4c75a825c70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733217245345 2024-12-03T09:14:08,327 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a26d235e06be44848d97af2120918a53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733217245631 2024-12-03T09:14:08,328 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46e2060b9e0b4e53a634123d8277cbdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733217246239 2024-12-03T09:14:08,338 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#C#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:08,339 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/36b2e786ec4a48849516968b79e68894 is 50, key is test_row_0/C:col10/1733217246239/Put/seqid=0 2024-12-03T09:14:08,352 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/807aa22b85eb4eb29875e637a38c7723 2024-12-03T09:14:08,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741922_1098 (size=13221) 2024-12-03T09:14:08,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/1f15775deb7f437f9653feb7c070aa08 is 50, key is test_row_0/C:col10/1733217247474/Put/seqid=0 2024-12-03T09:14:08,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741923_1099 (size=12301) 2024-12-03T09:14:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T09:14:08,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217308459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:08,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217308459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:08,767 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/1f15775deb7f437f9653feb7c070aa08 2024-12-03T09:14:08,767 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/36b2e786ec4a48849516968b79e68894 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/36b2e786ec4a48849516968b79e68894 2024-12-03T09:14:08,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217308769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:08,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217308769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:08,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/d44856af31b6457fbec7a019551c4f4c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/d44856af31b6457fbec7a019551c4f4c 2024-12-03T09:14:08,774 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/C of c49f86fc80f2af2d2b0cf03332e8cc12 into 36b2e786ec4a48849516968b79e68894(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:08,774 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:08,774 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/C, priority=12, startTime=1733217247875; duration=0sec 2024-12-03T09:14:08,774 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:08,774 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:C 2024-12-03T09:14:08,777 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/d44856af31b6457fbec7a019551c4f4c, entries=150, sequenceid=460, filesize=12.0 K 2024-12-03T09:14:08,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/807aa22b85eb4eb29875e637a38c7723 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/807aa22b85eb4eb29875e637a38c7723 2024-12-03T09:14:08,784 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/807aa22b85eb4eb29875e637a38c7723, entries=150, sequenceid=460, filesize=12.0 K 2024-12-03T09:14:08,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/1f15775deb7f437f9653feb7c070aa08 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/1f15775deb7f437f9653feb7c070aa08 2024-12-03T09:14:08,791 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/1f15775deb7f437f9653feb7c070aa08, entries=150, sequenceid=460, filesize=12.0 K 2024-12-03T09:14:08,792 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c49f86fc80f2af2d2b0cf03332e8cc12 in 866ms, sequenceid=460, compaction requested=false 2024-12-03T09:14:08,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:08,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:08,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-03T09:14:08,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-03T09:14:08,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-03T09:14:08,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0210 sec 2024-12-03T09:14:08,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.0270 sec 2024-12-03T09:14:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T09:14:08,909 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:08,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-12-03T09:14:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T09:14:08,912 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:08,913 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:08,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T09:14:09,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-03T09:14:09,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:09,066 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:14:09,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:09,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:09,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:09,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:09,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:09,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:09,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/a3a3223860114a15a844d693ae48cf08 is 50, key is test_row_0/A:col10/1733217248136/Put/seqid=0 2024-12-03T09:14:09,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741924_1100 (size=12301) 2024-12-03T09:14:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T09:14:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:09,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:09,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217249320, max timestamp=1733217249328Num uniques 859; 2024-12-03T09:14:09,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:14:09,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217249320, max timestamp=1733217249328Num uniques 859; 2024-12-03T09:14:09,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217249320, max timestamp=1733217249328Num uniques 859; 2024-12-03T09:14:09,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:14:09,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:14:09,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:09,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217309358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217309358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:09,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217309469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:09,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217309469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:09,481 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/a3a3223860114a15a844d693ae48cf08 2024-12-03T09:14:09,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/82e43cb51b924b1bbb2d0484d3fb429f is 50, key is test_row_0/B:col10/1733217248136/Put/seqid=0 2024-12-03T09:14:09,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741925_1101 (size=12301) 2024-12-03T09:14:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T09:14:09,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217309680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:09,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217309680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:09,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38734 deadline: 1733217309829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:09,834 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in c49f86fc80f2af2d2b0cf03332e8cc12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:09,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38762 deadline: 1733217309839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:09,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38732 deadline: 1733217309839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:09,841 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in c49f86fc80f2af2d2b0cf03332e8cc12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:09,841 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in c49f86fc80f2af2d2b0cf03332e8cc12 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:09,893 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/82e43cb51b924b1bbb2d0484d3fb429f 2024-12-03T09:14:09,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/47cd417796534d6983a24ff87e08fef4 is 50, key is test_row_0/C:col10/1733217248136/Put/seqid=0 2024-12-03T09:14:09,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741926_1102 (size=12301) 2024-12-03T09:14:09,911 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/47cd417796534d6983a24ff87e08fef4 2024-12-03T09:14:09,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/a3a3223860114a15a844d693ae48cf08 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/a3a3223860114a15a844d693ae48cf08 2024-12-03T09:14:09,923 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/a3a3223860114a15a844d693ae48cf08, entries=150, sequenceid=473, filesize=12.0 K 2024-12-03T09:14:09,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/82e43cb51b924b1bbb2d0484d3fb429f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/82e43cb51b924b1bbb2d0484d3fb429f 2024-12-03T09:14:09,935 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/82e43cb51b924b1bbb2d0484d3fb429f, entries=150, sequenceid=473, filesize=12.0 K 2024-12-03T09:14:09,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/47cd417796534d6983a24ff87e08fef4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/47cd417796534d6983a24ff87e08fef4 2024-12-03T09:14:09,947 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/47cd417796534d6983a24ff87e08fef4, entries=150, sequenceid=473, filesize=12.0 K 2024-12-03T09:14:09,948 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for c49f86fc80f2af2d2b0cf03332e8cc12 in 883ms, sequenceid=473, compaction requested=true 2024-12-03T09:14:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-03T09:14:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-03T09:14:09,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-03T09:14:09,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0370 sec 2024-12-03T09:14:09,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.0440 sec 2024-12-03T09:14:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:09,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-03T09:14:09,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733217249353, max timestamp=1733217249992Num uniques 1068; 2024-12-03T09:14:09,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T09:14:09,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733217249353, max timestamp=1733217249992Num uniques 1068; 2024-12-03T09:14:09,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733217249353, max timestamp=1733217249992Num uniques 1068; 2024-12-03T09:14:09,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T09:14:09,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T09:14:09,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:10,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/9ef8cb62bf014d5aa42f55a0f5a7c4e7 is 50, key is test_row_0/A:col10/1733217249353/Put/seqid=0 2024-12-03T09:14:10,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741927_1103 (size=14741) 2024-12-03T09:14:10,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733217250019, max timestamp=1733217250042Num uniques 859; 2024-12-03T09:14:10,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T09:14:10,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733217250019, max timestamp=1733217250042Num uniques 859; 2024-12-03T09:14:10,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T09:14:10,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733217250019, max timestamp=1733217250042Num uniques 859; 2024-12-03T09:14:10,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T09:14:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T09:14:10,057 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:10,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-12-03T09:14:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T09:14:10,061 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:10,062 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:10,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:10,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:10,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217310090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:10,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:10,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217310091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T09:14:10,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217310199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:10,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217310210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:10,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T09:14:10,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:10,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:10,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:10,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T09:14:10,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:10,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:10,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:10,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T09:14:10,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217310410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:10,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:10,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217310419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:10,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/9ef8cb62bf014d5aa42f55a0f5a7c4e7 2024-12-03T09:14:10,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f21d66944a69479a90ee00068940ba2c is 50, key is test_row_0/B:col10/1733217249353/Put/seqid=0 2024-12-03T09:14:10,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741928_1104 (size=12301) 2024-12-03T09:14:10,466 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-03T09:14:10,466 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-03T09:14:10,466 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:10,466 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:10,467 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-03T09:14:10,468 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:10,471 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:10,471 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:10,471 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:10,471 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:10,471 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:10,472 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:10,472 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:10,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:10,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:10,475 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-03T09:14:10,475 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:10,475 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:10,475 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:10,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:10,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T09:14:10,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:10,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:10,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:10,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T09:14:10,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:10,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:10,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:10,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T09:14:10,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217310720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:10,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:10,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217310729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:10,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T09:14:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:10,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:10,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:10,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,837 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35781 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=699c9baf3d29,42271,1733217224446, table=TestAcidGuarantees, region=c49f86fc80f2af2d2b0cf03332e8cc12. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T09:14:10,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f21d66944a69479a90ee00068940ba2c 2024-12-03T09:14:10,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/addbd22f7f984eedab766122b08c1da3 is 50, key is test_row_0/C:col10/1733217249353/Put/seqid=0 2024-12-03T09:14:10,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741929_1105 (size=12301) 2024-12-03T09:14:10,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T09:14:10,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:10,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:10,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:10,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:11,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T09:14:11,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:11,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:11,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:11,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:11,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T09:14:11,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38772 deadline: 1733217311230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:11,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:11,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:11,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:11,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38752 deadline: 1733217311239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:11,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:11,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:11,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:11,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/addbd22f7f984eedab766122b08c1da3 2024-12-03T09:14:11,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/9ef8cb62bf014d5aa42f55a0f5a7c4e7 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/9ef8cb62bf014d5aa42f55a0f5a7c4e7 2024-12-03T09:14:11,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/9ef8cb62bf014d5aa42f55a0f5a7c4e7, entries=200, sequenceid=512, filesize=14.4 K 2024-12-03T09:14:11,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/f21d66944a69479a90ee00068940ba2c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f21d66944a69479a90ee00068940ba2c 2024-12-03T09:14:11,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f21d66944a69479a90ee00068940ba2c, entries=150, sequenceid=512, filesize=12.0 K 2024-12-03T09:14:11,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/addbd22f7f984eedab766122b08c1da3 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/addbd22f7f984eedab766122b08c1da3 2024-12-03T09:14:11,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/addbd22f7f984eedab766122b08c1da3, entries=150, sequenceid=512, filesize=12.0 K 2024-12-03T09:14:11,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=154.31 KB/158010 for c49f86fc80f2af2d2b0cf03332e8cc12 in 1302ms, sequenceid=512, compaction requested=true 2024-12-03T09:14:11,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:11,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:14:11,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:11,296 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:11,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:14:11,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:11,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c49f86fc80f2af2d2b0cf03332e8cc12:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:14:11,296 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:11,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:11,297 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:11,298 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/B is initiating minor compaction (all files) 2024-12-03T09:14:11,298 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52564 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:11,298 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/B in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:11,298 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/A is initiating minor compaction (all files) 2024-12-03T09:14:11,298 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/A in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:11,298 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5df7792775f94dbe8b3122b5b1f38f99, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/807aa22b85eb4eb29875e637a38c7723, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/82e43cb51b924b1bbb2d0484d3fb429f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f21d66944a69479a90ee00068940ba2c] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=48.9 K 2024-12-03T09:14:11,298 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/e328c664b5b942039b99511e09fae7ca, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/d44856af31b6457fbec7a019551c4f4c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/a3a3223860114a15a844d693ae48cf08, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/9ef8cb62bf014d5aa42f55a0f5a7c4e7] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=51.3 K 2024-12-03T09:14:11,298 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5df7792775f94dbe8b3122b5b1f38f99, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733217246239 2024-12-03T09:14:11,298 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e328c664b5b942039b99511e09fae7ca, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733217246239 2024-12-03T09:14:11,298 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 807aa22b85eb4eb29875e637a38c7723, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733217247474 2024-12-03T09:14:11,299 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d44856af31b6457fbec7a019551c4f4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733217247474 2024-12-03T09:14:11,299 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 82e43cb51b924b1bbb2d0484d3fb429f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1733217248122 2024-12-03T09:14:11,299 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3a3223860114a15a844d693ae48cf08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1733217248122 2024-12-03T09:14:11,299 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f21d66944a69479a90ee00068940ba2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733217249353 2024-12-03T09:14:11,299 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ef8cb62bf014d5aa42f55a0f5a7c4e7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733217249353 2024-12-03T09:14:11,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-12-03T09:14:11,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:11,305 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-03T09:14:11,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:11,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:11,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:11,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:11,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:11,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:11,309 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#A#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:11,309 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:11,310 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/825356a599694e4ca8a12fc1bf2bb17b is 50, key is test_row_0/A:col10/1733217249353/Put/seqid=0 2024-12-03T09:14:11,310 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/666306f638da4729a48376697cdb1b09 is 50, key is test_row_0/B:col10/1733217249353/Put/seqid=0 2024-12-03T09:14:11,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/993b765a40de44e5abee7122c8650b72 is 50, key is test_row_0/A:col10/1733217250089/Put/seqid=0 2024-12-03T09:14:11,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741931_1107 (size=13357) 2024-12-03T09:14:11,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741930_1106 (size=13357) 2024-12-03T09:14:11,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741932_1108 (size=12301) 2024-12-03T09:14:11,716 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/993b765a40de44e5abee7122c8650b72 2024-12-03T09:14:11,725 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/825356a599694e4ca8a12fc1bf2bb17b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/825356a599694e4ca8a12fc1bf2bb17b 2024-12-03T09:14:11,725 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/666306f638da4729a48376697cdb1b09 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/666306f638da4729a48376697cdb1b09 2024-12-03T09:14:11,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/9a501a3f485b45098a7b29fbf89ed5ef is 50, key is test_row_0/B:col10/1733217250089/Put/seqid=0 2024-12-03T09:14:11,731 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/B of c49f86fc80f2af2d2b0cf03332e8cc12 into 666306f638da4729a48376697cdb1b09(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:11,731 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/A of c49f86fc80f2af2d2b0cf03332e8cc12 into 825356a599694e4ca8a12fc1bf2bb17b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:11,731 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:11,731 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:11,731 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/A, priority=12, startTime=1733217251296; duration=0sec 2024-12-03T09:14:11,731 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/B, priority=12, startTime=1733217251296; duration=0sec 2024-12-03T09:14:11,731 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:11,731 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:A 2024-12-03T09:14:11,731 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:11,731 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:11,731 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:B 2024-12-03T09:14:11,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741933_1109 (size=12301) 2024-12-03T09:14:11,733 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:11,733 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): c49f86fc80f2af2d2b0cf03332e8cc12/C is initiating minor compaction (all files) 2024-12-03T09:14:11,733 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c49f86fc80f2af2d2b0cf03332e8cc12/C in TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:11,733 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/36b2e786ec4a48849516968b79e68894, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/1f15775deb7f437f9653feb7c070aa08, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/47cd417796534d6983a24ff87e08fef4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/addbd22f7f984eedab766122b08c1da3] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp, totalSize=48.9 K 2024-12-03T09:14:11,733 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36b2e786ec4a48849516968b79e68894, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733217246239 2024-12-03T09:14:11,734 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f15775deb7f437f9653feb7c070aa08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1733217247474 2024-12-03T09:14:11,734 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47cd417796534d6983a24ff87e08fef4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1733217248122 2024-12-03T09:14:11,734 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting addbd22f7f984eedab766122b08c1da3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733217249353 2024-12-03T09:14:11,742 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c49f86fc80f2af2d2b0cf03332e8cc12#C#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:11,743 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/35be87307d1447ae95358b2e80a021bc is 50, key is test_row_0/C:col10/1733217249353/Put/seqid=0 2024-12-03T09:14:11,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741934_1110 (size=13357) 2024-12-03T09:14:12,135 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/9a501a3f485b45098a7b29fbf89ed5ef 2024-12-03T09:14:12,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/85228ece67734dfc8e1c4e2971dbf6d4 is 50, key is test_row_0/C:col10/1733217250089/Put/seqid=0 2024-12-03T09:14:12,155 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/35be87307d1447ae95358b2e80a021bc as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/35be87307d1447ae95358b2e80a021bc 2024-12-03T09:14:12,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741935_1111 (size=12301) 2024-12-03T09:14:12,161 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c49f86fc80f2af2d2b0cf03332e8cc12/C of c49f86fc80f2af2d2b0cf03332e8cc12 into 35be87307d1447ae95358b2e80a021bc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:12,161 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:12,161 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., storeName=c49f86fc80f2af2d2b0cf03332e8cc12/C, priority=12, startTime=1733217251296; duration=0sec 2024-12-03T09:14:12,161 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:12,161 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c49f86fc80f2af2d2b0cf03332e8cc12:C 2024-12-03T09:14:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T09:14:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:12,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. as already flushing 2024-12-03T09:14:12,243 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-03T09:14:12,243 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:12,243 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:12,243 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:12,243 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:12,253 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-03T09:14:12,253 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:12,253 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:12,253 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:12,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:12,558 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/85228ece67734dfc8e1c4e2971dbf6d4 2024-12-03T09:14:12,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/993b765a40de44e5abee7122c8650b72 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/993b765a40de44e5abee7122c8650b72 2024-12-03T09:14:12,576 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/993b765a40de44e5abee7122c8650b72, entries=150, sequenceid=549, filesize=12.0 K 2024-12-03T09:14:12,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/9a501a3f485b45098a7b29fbf89ed5ef as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/9a501a3f485b45098a7b29fbf89ed5ef 2024-12-03T09:14:12,582 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/9a501a3f485b45098a7b29fbf89ed5ef, entries=150, sequenceid=549, filesize=12.0 K 2024-12-03T09:14:12,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/85228ece67734dfc8e1c4e2971dbf6d4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/85228ece67734dfc8e1c4e2971dbf6d4 2024-12-03T09:14:12,587 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/85228ece67734dfc8e1c4e2971dbf6d4, entries=150, sequenceid=549, filesize=12.0 K 2024-12-03T09:14:12,588 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=13.42 KB/13740 for c49f86fc80f2af2d2b0cf03332e8cc12 in 1283ms, sequenceid=549, compaction requested=false 2024-12-03T09:14:12,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for c49f86fc80f2af2d2b0cf03332e8cc12: 2024-12-03T09:14:12,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:12,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-12-03T09:14:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-12-03T09:14:12,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T09:14:12,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5280 sec 2024-12-03T09:14:12,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 2.5350 sec 2024-12-03T09:14:12,799 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T09:14:13,918 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-03T09:14:13,918 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-03T09:14:13,918 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-03T09:14:13,918 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:13,918 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:13,918 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:13,918 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:13,918 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:13,918 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:13,918 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:13,918 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:13,918 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:13,918 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:13,918 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:13,919 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T09:14:14,241 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:14,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T09:14:14,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-03T09:14:14,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-12-03T09:14:14,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 128 2024-12-03T09:14:14,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-12-03T09:14:14,242 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-12-03T09:14:14,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T09:14:14,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3766 2024-12-03T09:14:14,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3908 2024-12-03T09:14:14,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T09:14:14,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1713 2024-12-03T09:14:14,243 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5139 rows 2024-12-03T09:14:14,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1733 2024-12-03T09:14:14,244 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5190 rows 2024-12-03T09:14:14,244 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T09:14:14,244 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:14:14,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:14,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:14,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:14,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T09:14:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T09:14:14,262 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217254262"}]},"ts":"1733217254262"} 2024-12-03T09:14:14,264 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T09:14:14,264 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T09:14:14,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T09:14:14,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49f86fc80f2af2d2b0cf03332e8cc12, UNASSIGN}] 2024-12-03T09:14:14,272 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49f86fc80f2af2d2b0cf03332e8cc12, UNASSIGN 2024-12-03T09:14:14,273 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=c49f86fc80f2af2d2b0cf03332e8cc12, regionState=CLOSING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:14,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49f86fc80f2af2d2b0cf03332e8cc12, UNASSIGN because future has completed 2024-12-03T09:14:14,276 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T09:14:14,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:14:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T09:14:14,433 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:14,433 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T09:14:14,434 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing c49f86fc80f2af2d2b0cf03332e8cc12, disabling compactions & flushes 2024-12-03T09:14:14,434 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:14,434 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:14,434 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. after waiting 0 ms 2024-12-03T09:14:14,434 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:14,435 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing c49f86fc80f2af2d2b0cf03332e8cc12 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T09:14:14,435 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=A 2024-12-03T09:14:14,435 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:14,435 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=B 2024-12-03T09:14:14,435 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:14,435 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c49f86fc80f2af2d2b0cf03332e8cc12, store=C 2024-12-03T09:14:14,435 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:14,441 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/fcb333c180dd4001be8e9fe898e19ce5 is 50, key is test_row_0/A:col10/1733217253879/Put/seqid=0 2024-12-03T09:14:14,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741936_1112 (size=12301) 2024-12-03T09:14:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T09:14:14,846 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/fcb333c180dd4001be8e9fe898e19ce5 2024-12-03T09:14:14,857 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/8e10af0ed70b469c8e6add19ad8a33aa is 50, key is test_row_0/B:col10/1733217253879/Put/seqid=0 2024-12-03T09:14:14,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741937_1113 (size=12301) 2024-12-03T09:14:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T09:14:15,262 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/8e10af0ed70b469c8e6add19ad8a33aa 2024-12-03T09:14:15,275 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/0a0a892333fe4e329ec0c1ef76b9c1af is 50, key is test_row_0/C:col10/1733217253879/Put/seqid=0 2024-12-03T09:14:15,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741938_1114 (size=12301) 2024-12-03T09:14:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T09:14:15,683 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/0a0a892333fe4e329ec0c1ef76b9c1af 2024-12-03T09:14:15,693 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/A/fcb333c180dd4001be8e9fe898e19ce5 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/fcb333c180dd4001be8e9fe898e19ce5 2024-12-03T09:14:15,699 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/fcb333c180dd4001be8e9fe898e19ce5, entries=150, sequenceid=560, filesize=12.0 K 2024-12-03T09:14:15,700 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/B/8e10af0ed70b469c8e6add19ad8a33aa as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/8e10af0ed70b469c8e6add19ad8a33aa 2024-12-03T09:14:15,705 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/8e10af0ed70b469c8e6add19ad8a33aa, entries=150, sequenceid=560, filesize=12.0 K 2024-12-03T09:14:15,706 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/.tmp/C/0a0a892333fe4e329ec0c1ef76b9c1af as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/0a0a892333fe4e329ec0c1ef76b9c1af 2024-12-03T09:14:15,710 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/0a0a892333fe4e329ec0c1ef76b9c1af, entries=150, sequenceid=560, filesize=12.0 K 2024-12-03T09:14:15,711 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c49f86fc80f2af2d2b0cf03332e8cc12 in 1277ms, sequenceid=560, compaction requested=true 2024-12-03T09:14:15,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/5d9367b189674ce498fa2b9dd0a7b39c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/3e96c59532b14925a1f6f37cd3340905, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2bbefc43af994b148d278904fee16500, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/1fe0d613d20c4dd6b6f72c541d34d79b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/0ead33cad4af426ab4848954c282b161, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/60ed4bf82f0949569cced6aac2eb48b2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/5d3e901aa1df4160805e45f369789bc5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/48acb7b302f64e97838bee0d8f95056c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/faec359b6f0b492f8d9b3f45ab4eedbe, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/40e4620d7af445f3826b6b69acf3d356, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/e6d4952742754f1eb78c57f12230790a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/b05037b67536449faa5405e6157a31dd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/c8cdefd32e9c40d998a18c3a455fb85c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/b1ff4dce61fa4131a908018331eae695, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2171a3b30f824f5887c0605764fbacba, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/3f3813001c944a56a136c150da6cd0e4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/fbad328f1109480dbcb8faeeb59b4301, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/041625aae23a4319adc4cb9b77604bc2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/39e18fadd94d4c3c8215634732a19c21, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/8103aaa2fd914ed8a6d865f1a36db67d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/83cdfa0540ce402b8ccdfe31a21ef986, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/895c64172b97431c9d52029bc171be0d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/25886ac9e69f4204a66578d61c18b121, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/87653ba887cb48c5812fb00a8641c0e0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/ed6361c740f545f99fd53b6f2a6017a1, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/689e5336b1ca438ca1ca6b1f1812b49b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2b121a5637a1498eb30f77b021b780bc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/e328c664b5b942039b99511e09fae7ca, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/d44856af31b6457fbec7a019551c4f4c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/a3a3223860114a15a844d693ae48cf08, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/9ef8cb62bf014d5aa42f55a0f5a7c4e7] to archive 2024-12-03T09:14:15,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:14:15,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/5d9367b189674ce498fa2b9dd0a7b39c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/5d9367b189674ce498fa2b9dd0a7b39c 2024-12-03T09:14:15,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/3e96c59532b14925a1f6f37cd3340905 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/3e96c59532b14925a1f6f37cd3340905 2024-12-03T09:14:15,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2bbefc43af994b148d278904fee16500 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2bbefc43af994b148d278904fee16500 2024-12-03T09:14:15,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/1fe0d613d20c4dd6b6f72c541d34d79b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/1fe0d613d20c4dd6b6f72c541d34d79b 2024-12-03T09:14:15,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/0ead33cad4af426ab4848954c282b161 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/0ead33cad4af426ab4848954c282b161 2024-12-03T09:14:15,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/60ed4bf82f0949569cced6aac2eb48b2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/60ed4bf82f0949569cced6aac2eb48b2 2024-12-03T09:14:15,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/5d3e901aa1df4160805e45f369789bc5 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/5d3e901aa1df4160805e45f369789bc5 2024-12-03T09:14:15,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/48acb7b302f64e97838bee0d8f95056c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/48acb7b302f64e97838bee0d8f95056c 2024-12-03T09:14:15,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/faec359b6f0b492f8d9b3f45ab4eedbe to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/faec359b6f0b492f8d9b3f45ab4eedbe 2024-12-03T09:14:15,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/40e4620d7af445f3826b6b69acf3d356 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/40e4620d7af445f3826b6b69acf3d356 2024-12-03T09:14:15,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/e6d4952742754f1eb78c57f12230790a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/e6d4952742754f1eb78c57f12230790a 2024-12-03T09:14:15,736 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/b05037b67536449faa5405e6157a31dd to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/b05037b67536449faa5405e6157a31dd 2024-12-03T09:14:15,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/c8cdefd32e9c40d998a18c3a455fb85c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/c8cdefd32e9c40d998a18c3a455fb85c 2024-12-03T09:14:15,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/b1ff4dce61fa4131a908018331eae695 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/b1ff4dce61fa4131a908018331eae695 2024-12-03T09:14:15,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2171a3b30f824f5887c0605764fbacba to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2171a3b30f824f5887c0605764fbacba 2024-12-03T09:14:15,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/3f3813001c944a56a136c150da6cd0e4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/3f3813001c944a56a136c150da6cd0e4 2024-12-03T09:14:15,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/fbad328f1109480dbcb8faeeb59b4301 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/fbad328f1109480dbcb8faeeb59b4301 2024-12-03T09:14:15,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/041625aae23a4319adc4cb9b77604bc2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/041625aae23a4319adc4cb9b77604bc2 2024-12-03T09:14:15,744 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/39e18fadd94d4c3c8215634732a19c21 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/39e18fadd94d4c3c8215634732a19c21 2024-12-03T09:14:15,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/8103aaa2fd914ed8a6d865f1a36db67d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/8103aaa2fd914ed8a6d865f1a36db67d 2024-12-03T09:14:15,746 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/83cdfa0540ce402b8ccdfe31a21ef986 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/83cdfa0540ce402b8ccdfe31a21ef986 2024-12-03T09:14:15,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/895c64172b97431c9d52029bc171be0d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/895c64172b97431c9d52029bc171be0d 2024-12-03T09:14:15,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/25886ac9e69f4204a66578d61c18b121 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/25886ac9e69f4204a66578d61c18b121 2024-12-03T09:14:15,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/87653ba887cb48c5812fb00a8641c0e0 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/87653ba887cb48c5812fb00a8641c0e0 2024-12-03T09:14:15,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/ed6361c740f545f99fd53b6f2a6017a1 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/ed6361c740f545f99fd53b6f2a6017a1 2024-12-03T09:14:15,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/689e5336b1ca438ca1ca6b1f1812b49b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/689e5336b1ca438ca1ca6b1f1812b49b 2024-12-03T09:14:15,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2b121a5637a1498eb30f77b021b780bc to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/2b121a5637a1498eb30f77b021b780bc 2024-12-03T09:14:15,752 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/e328c664b5b942039b99511e09fae7ca to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/e328c664b5b942039b99511e09fae7ca 2024-12-03T09:14:15,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/d44856af31b6457fbec7a019551c4f4c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/d44856af31b6457fbec7a019551c4f4c 2024-12-03T09:14:15,754 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/a3a3223860114a15a844d693ae48cf08 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/a3a3223860114a15a844d693ae48cf08 2024-12-03T09:14:15,756 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/9ef8cb62bf014d5aa42f55a0f5a7c4e7 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/9ef8cb62bf014d5aa42f55a0f5a7c4e7 2024-12-03T09:14:15,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/374b98ebc5004100b844e84868c73d75, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/b655b1253dbd4e5e965f2e545aac2bfc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/574ff23901c447fda67b6d6c32e6011b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/ac360c91b0584c24bcc8b011a4afd9b8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/43565553c69a4f42941d60c674cd0335, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/7500b0684a234779bae58bb0ed1a16b9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/080ea6b4695a40ae98404cc6399766fc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/91c9a49fd65042cd843c4042c49c3283, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f63e45894a514051abf60a18abcd1ca2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/627cf1d12725413b88a17250cba2910c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f0e8aae48b7340ba89638f24844fe83e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5deee6304de1486fbbec0ea4e0650bb9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f9387b855f844d1f83959d1da5199fbf, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/3122bf86ffca4c70916390a8148c7db2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/2e69360d9dec48528694bfdbc56c1cf2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/aac21e60572740f19d6e748d274469bc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/47078ddae9a74d5dbf46b9748a7f8404, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5d15cd599d8c4593bffe00d5dfb229ef, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/4232e4b93b544949ac36de04c8996884, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/6a47557d246b45ef8a36918556d93df5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/a5be33be05ad41c191d781364cdd616f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/0a4ae41b75a144aa9a1d607f1cf3c01d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f364ae6e9f7e4b9bb89df652b916ec7e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/b0922ee532764cf19c7f9fcc0f4b4132, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/931983e52ccf45e894a9e15c9c40fd39, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f42b79c5c6e649c09641e93845039832, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5df7792775f94dbe8b3122b5b1f38f99, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/579c7a79c66946219951b3738c0c13e3, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/807aa22b85eb4eb29875e637a38c7723, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/82e43cb51b924b1bbb2d0484d3fb429f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f21d66944a69479a90ee00068940ba2c] to archive 2024-12-03T09:14:15,769 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:14:15,770 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/374b98ebc5004100b844e84868c73d75 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/374b98ebc5004100b844e84868c73d75 2024-12-03T09:14:15,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/b655b1253dbd4e5e965f2e545aac2bfc to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/b655b1253dbd4e5e965f2e545aac2bfc 2024-12-03T09:14:15,773 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/574ff23901c447fda67b6d6c32e6011b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/574ff23901c447fda67b6d6c32e6011b 2024-12-03T09:14:15,774 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/ac360c91b0584c24bcc8b011a4afd9b8 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/ac360c91b0584c24bcc8b011a4afd9b8 2024-12-03T09:14:15,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/43565553c69a4f42941d60c674cd0335 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/43565553c69a4f42941d60c674cd0335 2024-12-03T09:14:15,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/7500b0684a234779bae58bb0ed1a16b9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/7500b0684a234779bae58bb0ed1a16b9 2024-12-03T09:14:15,777 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/080ea6b4695a40ae98404cc6399766fc to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/080ea6b4695a40ae98404cc6399766fc 2024-12-03T09:14:15,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/91c9a49fd65042cd843c4042c49c3283 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/91c9a49fd65042cd843c4042c49c3283 2024-12-03T09:14:15,779 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f63e45894a514051abf60a18abcd1ca2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f63e45894a514051abf60a18abcd1ca2 2024-12-03T09:14:15,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/627cf1d12725413b88a17250cba2910c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/627cf1d12725413b88a17250cba2910c 2024-12-03T09:14:15,782 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f0e8aae48b7340ba89638f24844fe83e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f0e8aae48b7340ba89638f24844fe83e 2024-12-03T09:14:15,783 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5deee6304de1486fbbec0ea4e0650bb9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5deee6304de1486fbbec0ea4e0650bb9 2024-12-03T09:14:15,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f9387b855f844d1f83959d1da5199fbf to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f9387b855f844d1f83959d1da5199fbf 2024-12-03T09:14:15,785 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/3122bf86ffca4c70916390a8148c7db2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/3122bf86ffca4c70916390a8148c7db2 2024-12-03T09:14:15,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/2e69360d9dec48528694bfdbc56c1cf2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/2e69360d9dec48528694bfdbc56c1cf2 2024-12-03T09:14:15,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/aac21e60572740f19d6e748d274469bc to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/aac21e60572740f19d6e748d274469bc 2024-12-03T09:14:15,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/47078ddae9a74d5dbf46b9748a7f8404 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/47078ddae9a74d5dbf46b9748a7f8404 2024-12-03T09:14:15,789 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5d15cd599d8c4593bffe00d5dfb229ef to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5d15cd599d8c4593bffe00d5dfb229ef 2024-12-03T09:14:15,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/4232e4b93b544949ac36de04c8996884 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/4232e4b93b544949ac36de04c8996884 2024-12-03T09:14:15,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/6a47557d246b45ef8a36918556d93df5 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/6a47557d246b45ef8a36918556d93df5 2024-12-03T09:14:15,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/a5be33be05ad41c191d781364cdd616f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/a5be33be05ad41c191d781364cdd616f 2024-12-03T09:14:15,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/0a4ae41b75a144aa9a1d607f1cf3c01d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/0a4ae41b75a144aa9a1d607f1cf3c01d 2024-12-03T09:14:15,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f364ae6e9f7e4b9bb89df652b916ec7e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f364ae6e9f7e4b9bb89df652b916ec7e 2024-12-03T09:14:15,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/b0922ee532764cf19c7f9fcc0f4b4132 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/b0922ee532764cf19c7f9fcc0f4b4132 2024-12-03T09:14:15,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/931983e52ccf45e894a9e15c9c40fd39 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/931983e52ccf45e894a9e15c9c40fd39 2024-12-03T09:14:15,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f42b79c5c6e649c09641e93845039832 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f42b79c5c6e649c09641e93845039832 2024-12-03T09:14:15,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5df7792775f94dbe8b3122b5b1f38f99 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/5df7792775f94dbe8b3122b5b1f38f99 2024-12-03T09:14:15,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/579c7a79c66946219951b3738c0c13e3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/579c7a79c66946219951b3738c0c13e3 2024-12-03T09:14:15,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/807aa22b85eb4eb29875e637a38c7723 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/807aa22b85eb4eb29875e637a38c7723 2024-12-03T09:14:15,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/82e43cb51b924b1bbb2d0484d3fb429f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/82e43cb51b924b1bbb2d0484d3fb429f 2024-12-03T09:14:15,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f21d66944a69479a90ee00068940ba2c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/f21d66944a69479a90ee00068940ba2c 2024-12-03T09:14:15,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b1dc583720c04e138f0ecf6c9e61c804, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/bcbd408323b74df0ba383f2e736237b8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/9a48c8f681f946aaa19de8ab6948e60c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/ac4e9bce9a2b47508aad69922e29a0de, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/a21d0c3a756f49f0941b177179c8a89b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/adcf3681a6ad4a0bba3d253ebb86e6bf, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/3daaaa8c3d684e1ba9ef3fe08d1d30c8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/4747da28b42348b19d70d514f04d5d5a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b8deec7d65c144d6abb7c707a1c397ae, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/3a6e898e546b4a049e2ee02764cac616, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/f4b91cc0172e493680f351e30d56b3d4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/7827aa7530404718ba870beec90c900b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/dedc615526194f48b7f3b085e0e83618, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b712e0f87297498bb4b84e37c9dc0ec0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/fd05d272228d4e40bb4b4b88c894cbc5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/00dd2f19d781476bb25eb8df574d4ff8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/5e5d82b4f72c4342af0f44a33a8ccee1, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/6406c36a5a234899bcd1a699d91673c5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/456a13fc1a344294bf2a8ce62e6177df, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/e8e24b1f522d4086bd4c2a5a8f6d4902, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/d0f8bc9c916047439464f51526ef9e8f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/8ede13d4b1e943109278d5d071f05b21, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/f83a591484f44d209678b097d8e6ebdd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/77aa87052d634da2844e8a551d872b4a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/0dcd854fd8c04b2192d4b4c75a825c70, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/a26d235e06be44848d97af2120918a53, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/36b2e786ec4a48849516968b79e68894, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/46e2060b9e0b4e53a634123d8277cbdf, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/1f15775deb7f437f9653feb7c070aa08, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/47cd417796534d6983a24ff87e08fef4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/addbd22f7f984eedab766122b08c1da3] to archive 2024-12-03T09:14:15,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:14:15,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b1dc583720c04e138f0ecf6c9e61c804 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b1dc583720c04e138f0ecf6c9e61c804 2024-12-03T09:14:15,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/bcbd408323b74df0ba383f2e736237b8 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/bcbd408323b74df0ba383f2e736237b8 2024-12-03T09:14:15,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/9a48c8f681f946aaa19de8ab6948e60c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/9a48c8f681f946aaa19de8ab6948e60c 2024-12-03T09:14:15,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/ac4e9bce9a2b47508aad69922e29a0de to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/ac4e9bce9a2b47508aad69922e29a0de 2024-12-03T09:14:15,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/a21d0c3a756f49f0941b177179c8a89b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/a21d0c3a756f49f0941b177179c8a89b 2024-12-03T09:14:15,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/adcf3681a6ad4a0bba3d253ebb86e6bf to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/adcf3681a6ad4a0bba3d253ebb86e6bf 2024-12-03T09:14:15,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/3daaaa8c3d684e1ba9ef3fe08d1d30c8 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/3daaaa8c3d684e1ba9ef3fe08d1d30c8 2024-12-03T09:14:15,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/4747da28b42348b19d70d514f04d5d5a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/4747da28b42348b19d70d514f04d5d5a 2024-12-03T09:14:15,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b8deec7d65c144d6abb7c707a1c397ae to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b8deec7d65c144d6abb7c707a1c397ae 2024-12-03T09:14:15,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/3a6e898e546b4a049e2ee02764cac616 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/3a6e898e546b4a049e2ee02764cac616 2024-12-03T09:14:15,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/f4b91cc0172e493680f351e30d56b3d4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/f4b91cc0172e493680f351e30d56b3d4 2024-12-03T09:14:15,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/7827aa7530404718ba870beec90c900b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/7827aa7530404718ba870beec90c900b 2024-12-03T09:14:15,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/dedc615526194f48b7f3b085e0e83618 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/dedc615526194f48b7f3b085e0e83618 2024-12-03T09:14:15,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b712e0f87297498bb4b84e37c9dc0ec0 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/b712e0f87297498bb4b84e37c9dc0ec0 2024-12-03T09:14:15,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/fd05d272228d4e40bb4b4b88c894cbc5 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/fd05d272228d4e40bb4b4b88c894cbc5 2024-12-03T09:14:15,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/00dd2f19d781476bb25eb8df574d4ff8 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/00dd2f19d781476bb25eb8df574d4ff8 2024-12-03T09:14:15,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/5e5d82b4f72c4342af0f44a33a8ccee1 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/5e5d82b4f72c4342af0f44a33a8ccee1 2024-12-03T09:14:15,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/6406c36a5a234899bcd1a699d91673c5 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/6406c36a5a234899bcd1a699d91673c5 2024-12-03T09:14:15,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/456a13fc1a344294bf2a8ce62e6177df to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/456a13fc1a344294bf2a8ce62e6177df 2024-12-03T09:14:15,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/e8e24b1f522d4086bd4c2a5a8f6d4902 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/e8e24b1f522d4086bd4c2a5a8f6d4902 2024-12-03T09:14:15,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/d0f8bc9c916047439464f51526ef9e8f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/d0f8bc9c916047439464f51526ef9e8f 2024-12-03T09:14:15,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/8ede13d4b1e943109278d5d071f05b21 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/8ede13d4b1e943109278d5d071f05b21 2024-12-03T09:14:15,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/f83a591484f44d209678b097d8e6ebdd to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/f83a591484f44d209678b097d8e6ebdd 2024-12-03T09:14:15,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/77aa87052d634da2844e8a551d872b4a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/77aa87052d634da2844e8a551d872b4a 2024-12-03T09:14:15,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/0dcd854fd8c04b2192d4b4c75a825c70 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/0dcd854fd8c04b2192d4b4c75a825c70 2024-12-03T09:14:15,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/a26d235e06be44848d97af2120918a53 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/a26d235e06be44848d97af2120918a53 2024-12-03T09:14:15,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/36b2e786ec4a48849516968b79e68894 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/36b2e786ec4a48849516968b79e68894 2024-12-03T09:14:15,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/46e2060b9e0b4e53a634123d8277cbdf to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/46e2060b9e0b4e53a634123d8277cbdf 2024-12-03T09:14:15,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/1f15775deb7f437f9653feb7c070aa08 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/1f15775deb7f437f9653feb7c070aa08 2024-12-03T09:14:15,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/47cd417796534d6983a24ff87e08fef4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/47cd417796534d6983a24ff87e08fef4 2024-12-03T09:14:15,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/addbd22f7f984eedab766122b08c1da3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/addbd22f7f984eedab766122b08c1da3 2024-12-03T09:14:15,863 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/recovered.edits/563.seqid, newMaxSeqId=563, maxSeqId=1 2024-12-03T09:14:15,864 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. 2024-12-03T09:14:15,864 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for c49f86fc80f2af2d2b0cf03332e8cc12: Waiting for close lock at 1733217254434Running coprocessor pre-close hooks at 1733217254434Disabling compacts and flushes for region at 1733217254434Disabling writes for close at 1733217254434Obtaining lock to block concurrent updates at 1733217254435 (+1 ms)Preparing flush snapshotting stores in c49f86fc80f2af2d2b0cf03332e8cc12 at 1733217254435Finished memstore snapshotting TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733217254435Flushing stores of TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12. at 1733217254436 (+1 ms)Flushing c49f86fc80f2af2d2b0cf03332e8cc12/A: creating writer at 1733217254436Flushing c49f86fc80f2af2d2b0cf03332e8cc12/A: appending metadata at 1733217254440 (+4 ms)Flushing c49f86fc80f2af2d2b0cf03332e8cc12/A: closing flushed file at 1733217254440Flushing c49f86fc80f2af2d2b0cf03332e8cc12/B: creating writer at 1733217254852 (+412 ms)Flushing c49f86fc80f2af2d2b0cf03332e8cc12/B: appending metadata at 1733217254856 (+4 ms)Flushing c49f86fc80f2af2d2b0cf03332e8cc12/B: closing flushed file at 1733217254856Flushing c49f86fc80f2af2d2b0cf03332e8cc12/C: creating writer at 1733217255271 (+415 ms)Flushing c49f86fc80f2af2d2b0cf03332e8cc12/C: appending metadata at 1733217255275 (+4 ms)Flushing c49f86fc80f2af2d2b0cf03332e8cc12/C: closing flushed file at 1733217255275Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@318d5d72: reopening flushed file at 1733217255692 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@df2cadc: reopening flushed file at 1733217255699 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d83e10d: reopening flushed file at 1733217255705 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c49f86fc80f2af2d2b0cf03332e8cc12 in 1277ms, sequenceid=560, compaction requested=true at 1733217255711 (+6 ms)Writing region close event to WAL at 1733217255860 (+149 ms)Running coprocessor post-close hooks at 1733217255864 (+4 ms)Closed at 1733217255864 2024-12-03T09:14:15,866 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:15,867 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=c49f86fc80f2af2d2b0cf03332e8cc12, regionState=CLOSED 2024-12-03T09:14:15,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:14:15,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-03T09:14:15,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure c49f86fc80f2af2d2b0cf03332e8cc12, server=699c9baf3d29,42271,1733217224446 in 1.5930 sec 2024-12-03T09:14:15,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-03T09:14:15,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c49f86fc80f2af2d2b0cf03332e8cc12, UNASSIGN in 1.6000 sec 2024-12-03T09:14:15,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T09:14:15,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6070 sec 2024-12-03T09:14:15,875 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217255875"}]},"ts":"1733217255875"} 2024-12-03T09:14:15,877 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T09:14:15,877 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T09:14:15,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6240 sec 2024-12-03T09:14:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T09:14:16,420 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:16,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T09:14:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:16,436 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:16,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-03T09:14:16,438 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:16,442 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:16,446 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/recovered.edits] 2024-12-03T09:14:16,449 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/825356a599694e4ca8a12fc1bf2bb17b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/825356a599694e4ca8a12fc1bf2bb17b 2024-12-03T09:14:16,450 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/993b765a40de44e5abee7122c8650b72 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/993b765a40de44e5abee7122c8650b72 2024-12-03T09:14:16,451 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/fcb333c180dd4001be8e9fe898e19ce5 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/A/fcb333c180dd4001be8e9fe898e19ce5 2024-12-03T09:14:16,453 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/666306f638da4729a48376697cdb1b09 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/666306f638da4729a48376697cdb1b09 2024-12-03T09:14:16,455 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/8e10af0ed70b469c8e6add19ad8a33aa to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/8e10af0ed70b469c8e6add19ad8a33aa 2024-12-03T09:14:16,456 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/9a501a3f485b45098a7b29fbf89ed5ef to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/B/9a501a3f485b45098a7b29fbf89ed5ef 2024-12-03T09:14:16,458 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/0a0a892333fe4e329ec0c1ef76b9c1af to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/0a0a892333fe4e329ec0c1ef76b9c1af 2024-12-03T09:14:16,459 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/35be87307d1447ae95358b2e80a021bc to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/35be87307d1447ae95358b2e80a021bc 2024-12-03T09:14:16,460 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/85228ece67734dfc8e1c4e2971dbf6d4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/C/85228ece67734dfc8e1c4e2971dbf6d4 2024-12-03T09:14:16,463 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/recovered.edits/563.seqid to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12/recovered.edits/563.seqid 2024-12-03T09:14:16,463 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/c49f86fc80f2af2d2b0cf03332e8cc12 2024-12-03T09:14:16,463 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T09:14:16,468 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:16,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-03T09:14:16,477 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T09:14:16,504 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T09:14:16,505 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:16,505 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T09:14:16,505 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733217256505"}]},"ts":"9223372036854775807"} 2024-12-03T09:14:16,509 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T09:14:16,509 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c49f86fc80f2af2d2b0cf03332e8cc12, NAME => 'TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T09:14:16,509 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T09:14:16,509 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733217256509"}]},"ts":"9223372036854775807"} 2024-12-03T09:14:16,512 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T09:14:16,513 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:16,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 87 msec 2024-12-03T09:14:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-03T09:14:16,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T09:14:16,551 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:16,564 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1847676809_22 at /127.0.0.1:50694 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1209303677_22 at /127.0.0.1:38054 [Waiting for operation #301] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;699c9baf3d29:42271-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1209303677_22 at /127.0.0.1:37928 [Waiting for operation #308] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=341 (was 229) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=13136 (was 13714) 2024-12-03T09:14:16,574 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=341, ProcessCount=11, AvailableMemoryMB=13136 2024-12-03T09:14:16,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T09:14:16,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T09:14:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:16,579 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T09:14:16,579 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:16,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-12-03T09:14:16,580 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T09:14:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T09:14:16,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741939_1115 (size=1061) 2024-12-03T09:14:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T09:14:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T09:14:16,991 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09 2024-12-03T09:14:17,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741940_1116 (size=53) 2024-12-03T09:14:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T09:14:17,401 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:14:17,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7bd72df0dd602d0d382d15e837ff9272, disabling compactions & flushes 2024-12-03T09:14:17,402 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:17,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:17,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. after waiting 0 ms 2024-12-03T09:14:17,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:17,402 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:17,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7bd72df0dd602d0d382d15e837ff9272: Waiting for close lock at 1733217257402Disabling compacts and flushes for region at 1733217257402Disabling writes for close at 1733217257402Writing region close event to WAL at 1733217257402Closed at 1733217257402 2024-12-03T09:14:17,404 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T09:14:17,404 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733217257404"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733217257404"}]},"ts":"1733217257404"} 2024-12-03T09:14:17,408 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T09:14:17,409 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T09:14:17,410 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217257409"}]},"ts":"1733217257409"} 2024-12-03T09:14:17,412 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T09:14:17,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, ASSIGN}] 2024-12-03T09:14:17,414 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, ASSIGN 2024-12-03T09:14:17,415 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, ASSIGN; state=OFFLINE, location=699c9baf3d29,42271,1733217224446; forceNewPlan=false, retain=false 2024-12-03T09:14:17,567 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=7bd72df0dd602d0d382d15e837ff9272, regionState=OPENING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:17,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, ASSIGN because future has completed 2024-12-03T09:14:17,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:14:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T09:14:17,736 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:17,737 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:14:17,737 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:17,738 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:14:17,738 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:17,738 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:17,742 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:17,745 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:14:17,745 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bd72df0dd602d0d382d15e837ff9272 columnFamilyName A 2024-12-03T09:14:17,746 DEBUG [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:17,746 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.HStore(327): Store=7bd72df0dd602d0d382d15e837ff9272/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:14:17,746 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:17,748 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:14:17,748 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bd72df0dd602d0d382d15e837ff9272 columnFamilyName B 2024-12-03T09:14:17,748 DEBUG [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:17,749 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.HStore(327): Store=7bd72df0dd602d0d382d15e837ff9272/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:14:17,749 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:17,750 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:14:17,750 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bd72df0dd602d0d382d15e837ff9272 columnFamilyName C 2024-12-03T09:14:17,750 DEBUG [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:17,751 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.HStore(327): Store=7bd72df0dd602d0d382d15e837ff9272/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:14:17,751 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:17,751 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:17,752 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:17,752 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:17,753 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:17,753 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:17,754 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T09:14:17,755 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:17,757 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:14:17,758 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 7bd72df0dd602d0d382d15e837ff9272; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64335818, jitterRate=-0.04132160544395447}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T09:14:17,758 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:17,758 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 7bd72df0dd602d0d382d15e837ff9272: Running coprocessor pre-open hook at 1733217257738Writing region info on filesystem at 1733217257738Initializing all the Stores at 1733217257741 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217257741Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217257741Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217257741Cleaning up temporary data from old regions at 1733217257753 (+12 ms)Running coprocessor post-open hooks at 1733217257758 (+5 ms)Region opened successfully at 1733217257758 2024-12-03T09:14:17,759 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., pid=38, masterSystemTime=1733217257728 2024-12-03T09:14:17,761 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:17,761 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:17,761 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=7bd72df0dd602d0d382d15e837ff9272, regionState=OPEN, openSeqNum=2, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:17,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:14:17,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-03T09:14:17,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 in 190 msec 2024-12-03T09:14:17,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-03T09:14:17,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, ASSIGN in 352 msec 2024-12-03T09:14:17,768 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T09:14:17,768 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217257768"}]},"ts":"1733217257768"} 2024-12-03T09:14:17,769 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T09:14:17,770 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T09:14:17,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1940 sec 2024-12-03T09:14:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T09:14:18,730 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:18,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358abfb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:18,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:18,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:18,733 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:18,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4caf3862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:18,737 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cedc1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:18,739 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:18,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:18,746 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T09:14:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 699c9baf3d29,35781,1733217223715 2024-12-03T09:14:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17eb0dc9 2024-12-03T09:14:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T09:14:18,748 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T09:14:18,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T09:14:18,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T09:14:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:18,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741941_1117 (size=1097) 2024-12-03T09:14:19,232 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T09:14:19,232 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T09:14:19,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T09:14:19,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, REOPEN/MOVE}] 2024-12-03T09:14:19,249 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, REOPEN/MOVE 2024-12-03T09:14:19,250 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=7bd72df0dd602d0d382d15e837ff9272, regionState=CLOSING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:19,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, REOPEN/MOVE because future has completed 2024-12-03T09:14:19,252 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T09:14:19,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:14:19,405 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,405 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T09:14:19,406 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing 7bd72df0dd602d0d382d15e837ff9272, disabling compactions & flushes 2024-12-03T09:14:19,406 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:19,406 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:19,406 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. after waiting 0 ms 2024-12-03T09:14:19,406 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:19,415 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T09:14:19,416 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:19,416 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for 7bd72df0dd602d0d382d15e837ff9272: Waiting for close lock at 1733217259405Running coprocessor pre-close hooks at 1733217259405Disabling compacts and flushes for region at 1733217259405Disabling writes for close at 1733217259406 (+1 ms)Writing region close event to WAL at 1733217259407 (+1 ms)Running coprocessor post-close hooks at 1733217259416 (+9 ms)Closed at 1733217259416 2024-12-03T09:14:19,417 WARN [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: 7bd72df0dd602d0d382d15e837ff9272 to self. 2024-12-03T09:14:19,419 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,420 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=7bd72df0dd602d0d382d15e837ff9272, regionState=CLOSED 2024-12-03T09:14:19,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:14:19,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-12-03T09:14:19,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure 7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 in 171 msec 2024-12-03T09:14:19,425 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, REOPEN/MOVE; state=CLOSED, location=699c9baf3d29,42271,1733217224446; forceNewPlan=false, retain=true 2024-12-03T09:14:19,576 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=7bd72df0dd602d0d382d15e837ff9272, regionState=OPENING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:19,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, REOPEN/MOVE because future has completed 2024-12-03T09:14:19,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:14:19,738 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:19,738 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:14:19,738 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,738 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:14:19,738 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,738 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,742 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,743 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:14:19,748 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bd72df0dd602d0d382d15e837ff9272 columnFamilyName A 2024-12-03T09:14:19,750 DEBUG [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:19,750 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.HStore(327): Store=7bd72df0dd602d0d382d15e837ff9272/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:14:19,751 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,751 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:14:19,752 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bd72df0dd602d0d382d15e837ff9272 columnFamilyName B 2024-12-03T09:14:19,752 DEBUG [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:19,752 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.HStore(327): Store=7bd72df0dd602d0d382d15e837ff9272/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:14:19,752 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,753 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:14:19,753 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bd72df0dd602d0d382d15e837ff9272 columnFamilyName C 2024-12-03T09:14:19,753 DEBUG [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:19,753 INFO [StoreOpener-7bd72df0dd602d0d382d15e837ff9272-1 {}] regionserver.HStore(327): Store=7bd72df0dd602d0d382d15e837ff9272/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:14:19,753 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:19,753 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,754 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,755 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,756 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,756 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,756 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T09:14:19,757 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,758 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 7bd72df0dd602d0d382d15e837ff9272; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65889219, jitterRate=-0.018174126744270325}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T09:14:19,758 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,759 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 7bd72df0dd602d0d382d15e837ff9272: Running coprocessor pre-open hook at 1733217259738Writing region info on filesystem at 1733217259738Initializing all the Stores at 1733217259739 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217259739Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217259739Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217259739Cleaning up temporary data from old regions at 1733217259756 (+17 ms)Running coprocessor post-open hooks at 1733217259758 (+2 ms)Region opened successfully at 1733217259759 (+1 ms) 2024-12-03T09:14:19,760 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., pid=43, masterSystemTime=1733217259735 2024-12-03T09:14:19,762 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:19,762 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:19,762 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=7bd72df0dd602d0d382d15e837ff9272, regionState=OPEN, openSeqNum=5, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:19,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:14:19,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-12-03T09:14:19,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure 7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 in 183 msec 2024-12-03T09:14:19,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-03T09:14:19,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, REOPEN/MOVE in 519 msec 2024-12-03T09:14:19,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-03T09:14:19,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 533 msec 2024-12-03T09:14:19,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0190 sec 2024-12-03T09:14:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-03T09:14:19,777 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:19,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7492545c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:19,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:19,779 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6acf807a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:19,780 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:19,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d89034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d8bf6f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:19,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:19,782 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efd2437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:19,784 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@608b7395, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29477cab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:19,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:19,786 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf0f4de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:19,788 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a31414, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79051781, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:19,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:19,790 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1159fc15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:19,791 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d28b148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56f04185, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:19,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:19,793 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6385d5d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:19,795 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11725788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@239184b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:19,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:19,797 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b65cf6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:19,799 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:19,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b853e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca77b9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:19,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:19,800 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:19,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:19,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:19,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68a2829d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:19,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:19,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:19,802 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7203a827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bd04f44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:19,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:19,804 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bfb8bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:19,805 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7825eb66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23477936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,806 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:19,806 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:19,807 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:19,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:19,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:19,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c517d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:19,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:19,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:19,808 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411e8aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:19,810 DEBUG [Thread-616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:19,810 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:19,810 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:19,810 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:19,810 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:19,810 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:19,810 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:19,811 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:19,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-03T09:14:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:19,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:19,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T09:14:19,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:19,814 DEBUG [Thread-607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:19,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:19,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:19,815 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:19,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:19,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:19,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:19,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:19,816 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:19,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:19,816 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:19,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:19,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:19,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:19,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:19,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:14:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:14:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:14:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:14:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:14:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:14:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:14:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:14:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:14:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:14:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:19,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217259851, max timestamp=1733217259878Num uniques 859; 2024-12-03T09:14:19,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:19,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=609, cellSize=27885, totalHeapSize=63576, min timestamp=1733217259842, max timestamp=1733217259873Num uniques 859; 2024-12-03T09:14:19,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=27885, suffix heap size=117064,new segment heap size=63576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=609 2024-12-03T09:14:19,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217259851, max timestamp=1733217259878Num uniques 859; 2024-12-03T09:14:19,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:19,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031be4b09d948d477b9fb9c3562dea3e8f_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217259829/Put/seqid=0 2024-12-03T09:14:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T09:14:19,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217319928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:19,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:19,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:19,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:19,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217319931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:19,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1733217319934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:19,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:19,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741942_1118 (size=19474) 2024-12-03T09:14:19,946 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:19,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58108 deadline: 1733217319936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:19,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:19,953 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031be4b09d948d477b9fb9c3562dea3e8f_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031be4b09d948d477b9fb9c3562dea3e8f_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:19,955 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3fb94f46b0dc4c87b8ffc74711eebd7b, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:19,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58054 deadline: 1733217319939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:19,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3fb94f46b0dc4c87b8ffc74711eebd7b is 175, key is test_row_0/A:col10/1733217259829/Put/seqid=0 2024-12-03T09:14:19,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:19,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:19,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:19,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T09:14:19,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:19,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:19,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:19,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:19,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:19,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741943_1119 (size=56733) 2024-12-03T09:14:20,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217320039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217320050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1733217320050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58108 deadline: 1733217320059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58054 deadline: 1733217320070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T09:14:20,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:20,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:20,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:20,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T09:14:20,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217320250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217320259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1733217320271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T09:14:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:20,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58108 deadline: 1733217320279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58054 deadline: 1733217320280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,401 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3fb94f46b0dc4c87b8ffc74711eebd7b 2024-12-03T09:14:20,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/0c976b4a15f1454d93095fa4db0fa823 is 50, key is test_row_0/B:col10/1733217259829/Put/seqid=0 2024-12-03T09:14:20,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T09:14:20,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:20,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:20,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:20,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T09:14:20,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741944_1120 (size=12001) 2024-12-03T09:14:20,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217320560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217320569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1733217320579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T09:14:20,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:20,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:20,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:20,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58108 deadline: 1733217320590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58054 deadline: 1733217320590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:20,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T09:14:20,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:20,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:20,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:20,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/0c976b4a15f1454d93095fa4db0fa823 2024-12-03T09:14:20,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T09:14:20,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:20,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:20,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:20,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:20,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/e2e333af13184ee993ae09ec0abdf3d0 is 50, key is test_row_0/C:col10/1733217259829/Put/seqid=0 2024-12-03T09:14:20,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741945_1121 (size=12001) 2024-12-03T09:14:20,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/e2e333af13184ee993ae09ec0abdf3d0 2024-12-03T09:14:20,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3fb94f46b0dc4c87b8ffc74711eebd7b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3fb94f46b0dc4c87b8ffc74711eebd7b 2024-12-03T09:14:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T09:14:20,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3fb94f46b0dc4c87b8ffc74711eebd7b, entries=300, sequenceid=14, filesize=55.4 K 2024-12-03T09:14:20,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/0c976b4a15f1454d93095fa4db0fa823 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/0c976b4a15f1454d93095fa4db0fa823 2024-12-03T09:14:20,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/0c976b4a15f1454d93095fa4db0fa823, entries=150, sequenceid=14, filesize=11.7 K 2024-12-03T09:14:20,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/e2e333af13184ee993ae09ec0abdf3d0 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/e2e333af13184ee993ae09ec0abdf3d0 2024-12-03T09:14:20,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/e2e333af13184ee993ae09ec0abdf3d0, entries=150, sequenceid=14, filesize=11.7 K 2024-12-03T09:14:20,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 7bd72df0dd602d0d382d15e837ff9272 in 1151ms, sequenceid=14, compaction requested=false 2024-12-03T09:14:20,983 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T09:14:20,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:21,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T09:14:21,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:21,057 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-12-03T09:14:21,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:21,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:21,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:21,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:21,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:21,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:21,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:21,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c5c750be3b4340f5a2b7ef668daf5c2f_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217259933/Put/seqid=0 2024-12-03T09:14:21,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217321092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:21,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217321094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:21,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1733217321100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:21,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58108 deadline: 1733217321100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:21,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58054 deadline: 1733217321100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:21,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741946_1122 (size=12154) 2024-12-03T09:14:21,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:21,124 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c5c750be3b4340f5a2b7ef668daf5c2f_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c5c750be3b4340f5a2b7ef668daf5c2f_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:21,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/b8713a54239f4619b0f1b826f5b76fed, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:21,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/b8713a54239f4619b0f1b826f5b76fed is 175, key is test_row_0/A:col10/1733217259933/Put/seqid=0 2024-12-03T09:14:21,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741947_1123 (size=30955) 2024-12-03T09:14:21,152 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/b8713a54239f4619b0f1b826f5b76fed 2024-12-03T09:14:21,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/07d1ab28a5cc49fcb3ce335c3ae06c58 is 50, key is test_row_0/B:col10/1733217259933/Put/seqid=0 2024-12-03T09:14:21,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217321200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:21,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:21,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741948_1124 (size=12001) 2024-12-03T09:14:21,209 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/07d1ab28a5cc49fcb3ce335c3ae06c58 2024-12-03T09:14:21,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217321209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:21,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/20c33723c8aa4ea0ab158254d6824b05 is 50, key is test_row_0/C:col10/1733217259933/Put/seqid=0 2024-12-03T09:14:21,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741949_1125 (size=12001) 2024-12-03T09:14:21,412 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T09:14:21,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217321410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:21,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217321419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:21,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:21,635 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/20c33723c8aa4ea0ab158254d6824b05 2024-12-03T09:14:21,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/b8713a54239f4619b0f1b826f5b76fed as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/b8713a54239f4619b0f1b826f5b76fed 2024-12-03T09:14:21,658 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/b8713a54239f4619b0f1b826f5b76fed, entries=150, sequenceid=50, filesize=30.2 K 2024-12-03T09:14:21,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/07d1ab28a5cc49fcb3ce335c3ae06c58 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/07d1ab28a5cc49fcb3ce335c3ae06c58 2024-12-03T09:14:21,670 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/07d1ab28a5cc49fcb3ce335c3ae06c58, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T09:14:21,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/20c33723c8aa4ea0ab158254d6824b05 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/20c33723c8aa4ea0ab158254d6824b05 2024-12-03T09:14:21,685 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/20c33723c8aa4ea0ab158254d6824b05, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T09:14:21,686 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 7bd72df0dd602d0d382d15e837ff9272 in 629ms, sequenceid=50, compaction requested=false 2024-12-03T09:14:21,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:21,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:21,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-03T09:14:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-03T09:14:21,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-03T09:14:21,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8710 sec 2024-12-03T09:14:21,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.8810 sec 2024-12-03T09:14:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:21,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:14:21,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:21,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:21,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:21,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:21,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:21,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:21,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f8e10385c66e4b71adeae3147cd1f10f_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217261091/Put/seqid=0 2024-12-03T09:14:21,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741950_1126 (size=14594) 2024-12-03T09:14:21,770 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:21,776 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f8e10385c66e4b71adeae3147cd1f10f_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f8e10385c66e4b71adeae3147cd1f10f_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:21,780 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/12209cb8ec004ca1b285436b98a0cecf, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:21,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/12209cb8ec004ca1b285436b98a0cecf is 175, key is test_row_0/A:col10/1733217261091/Put/seqid=0 2024-12-03T09:14:21,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741951_1127 (size=39549) 2024-12-03T09:14:21,798 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/12209cb8ec004ca1b285436b98a0cecf 2024-12-03T09:14:21,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217261749, max timestamp=1733217261802Num uniques 859; 2024-12-03T09:14:21,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217261749, max timestamp=1733217261802Num uniques 859; 2024-12-03T09:14:21,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:14:21,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:14:21,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217261749, max timestamp=1733217261802Num uniques 859; 2024-12-03T09:14:21,808 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:14:21,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/3dbc6446009d47fab60625c7a03f88b9 is 50, key is test_row_0/B:col10/1733217261091/Put/seqid=0 2024-12-03T09:14:21,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741952_1128 (size=12001) 2024-12-03T09:14:21,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/3dbc6446009d47fab60625c7a03f88b9 2024-12-03T09:14:21,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217321861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:21,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217321863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:21,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/ee5f5e330a124569aa666d7bb91e81fe is 50, key is test_row_0/C:col10/1733217261091/Put/seqid=0 2024-12-03T09:14:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:21,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741953_1129 (size=12001) 2024-12-03T09:14:21,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/ee5f5e330a124569aa666d7bb91e81fe 2024-12-03T09:14:21,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/12209cb8ec004ca1b285436b98a0cecf as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/12209cb8ec004ca1b285436b98a0cecf 2024-12-03T09:14:21,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/12209cb8ec004ca1b285436b98a0cecf, entries=200, sequenceid=60, filesize=38.6 K 2024-12-03T09:14:21,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/3dbc6446009d47fab60625c7a03f88b9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/3dbc6446009d47fab60625c7a03f88b9 2024-12-03T09:14:21,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/3dbc6446009d47fab60625c7a03f88b9, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T09:14:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/ee5f5e330a124569aa666d7bb91e81fe as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/ee5f5e330a124569aa666d7bb91e81fe 2024-12-03T09:14:21,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/ee5f5e330a124569aa666d7bb91e81fe, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T09:14:21,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7bd72df0dd602d0d382d15e837ff9272 in 238ms, sequenceid=60, compaction requested=true 2024-12-03T09:14:21,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:21,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:14:21,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:21,961 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:21,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:14:21,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:21,961 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:21,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:14:21,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:21,962 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:21,962 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:21,962 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/A is initiating minor compaction (all files) 2024-12-03T09:14:21,962 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/B is initiating minor compaction (all files) 2024-12-03T09:14:21,963 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/B in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:21,963 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/A in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:21,963 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/0c976b4a15f1454d93095fa4db0fa823, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/07d1ab28a5cc49fcb3ce335c3ae06c58, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/3dbc6446009d47fab60625c7a03f88b9] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=35.2 K 2024-12-03T09:14:21,963 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3fb94f46b0dc4c87b8ffc74711eebd7b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/b8713a54239f4619b0f1b826f5b76fed, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/12209cb8ec004ca1b285436b98a0cecf] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=124.3 K 2024-12-03T09:14:21,963 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:21,963 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3fb94f46b0dc4c87b8ffc74711eebd7b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/b8713a54239f4619b0f1b826f5b76fed, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/12209cb8ec004ca1b285436b98a0cecf] 2024-12-03T09:14:21,964 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c976b4a15f1454d93095fa4db0fa823, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733217259824 2024-12-03T09:14:21,964 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fb94f46b0dc4c87b8ffc74711eebd7b, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733217259822 2024-12-03T09:14:21,964 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 07d1ab28a5cc49fcb3ce335c3ae06c58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733217259920 2024-12-03T09:14:21,964 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dbc6446009d47fab60625c7a03f88b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733217261082 2024-12-03T09:14:21,964 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8713a54239f4619b0f1b826f5b76fed, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733217259920 2024-12-03T09:14:21,965 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12209cb8ec004ca1b285436b98a0cecf, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733217261082 2024-12-03T09:14:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T09:14:21,970 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:21,971 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-03T09:14:21,977 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:21,977 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:21,978 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:21,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T09:14:21,978 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/088efacc166a47f484d28645e36c779c is 50, key is test_row_0/B:col10/1733217261091/Put/seqid=0 2024-12-03T09:14:21,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T09:14:21,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:21,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:21,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:21,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:21,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:21,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:21,982 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:21,997 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120365e2cd3fa6f04ce2938434f22b32b4e6_7bd72df0dd602d0d382d15e837ff9272 store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:22,000 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120365e2cd3fa6f04ce2938434f22b32b4e6_7bd72df0dd602d0d382d15e837ff9272, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:22,000 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120365e2cd3fa6f04ce2938434f22b32b4e6_7bd72df0dd602d0d382d15e837ff9272 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:22,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741954_1130 (size=12104) 2024-12-03T09:14:22,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203decd3811801047aea138e64bfa2910f4_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217261854/Put/seqid=0 2024-12-03T09:14:22,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:22,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217322024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217322025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:22,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:22,033 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/088efacc166a47f484d28645e36c779c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/088efacc166a47f484d28645e36c779c 2024-12-03T09:14:22,042 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/B of 7bd72df0dd602d0d382d15e837ff9272 into 088efacc166a47f484d28645e36c779c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:22,042 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:22,042 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/B, priority=13, startTime=1733217261961; duration=0sec 2024-12-03T09:14:22,042 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:22,042 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:B 2024-12-03T09:14:22,042 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:22,045 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:22,045 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/C is initiating minor compaction (all files) 2024-12-03T09:14:22,045 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/C in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:22,046 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/e2e333af13184ee993ae09ec0abdf3d0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/20c33723c8aa4ea0ab158254d6824b05, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/ee5f5e330a124569aa666d7bb91e81fe] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=35.2 K 2024-12-03T09:14:22,046 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting e2e333af13184ee993ae09ec0abdf3d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733217259824 2024-12-03T09:14:22,047 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 20c33723c8aa4ea0ab158254d6824b05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733217259920 2024-12-03T09:14:22,047 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5f5e330a124569aa666d7bb91e81fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733217261082 2024-12-03T09:14:22,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741955_1131 (size=4469) 2024-12-03T09:14:22,054 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#A#compaction#112 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:22,054 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/185858289cf3430da4505d7eca519f86 is 175, key is test_row_0/A:col10/1733217261091/Put/seqid=0 2024-12-03T09:14:22,073 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#C#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:22,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741956_1132 (size=12154) 2024-12-03T09:14:22,076 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:22,076 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/bc8b32bc223e4b7a80dc8bd03a67b703 is 50, key is test_row_0/C:col10/1733217261091/Put/seqid=0 2024-12-03T09:14:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T09:14:22,084 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203decd3811801047aea138e64bfa2910f4_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203decd3811801047aea138e64bfa2910f4_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:22,085 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/fc9125b805fa4ab799fc7163ee750c90, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:22,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/fc9125b805fa4ab799fc7163ee750c90 is 175, key is test_row_0/A:col10/1733217261854/Put/seqid=0 2024-12-03T09:14:22,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741957_1133 (size=31058) 2024-12-03T09:14:22,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741958_1134 (size=12104) 2024-12-03T09:14:22,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741959_1135 (size=30955) 2024-12-03T09:14:22,100 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/fc9125b805fa4ab799fc7163ee750c90 2024-12-03T09:14:22,107 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/bc8b32bc223e4b7a80dc8bd03a67b703 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/bc8b32bc223e4b7a80dc8bd03a67b703 2024-12-03T09:14:22,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/34698cf1ac0c48948fe68e964e5abde0 is 50, key is test_row_0/B:col10/1733217261854/Put/seqid=0 2024-12-03T09:14:22,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1733217322109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:22,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58108 deadline: 1733217322110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:22,115 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:22,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:22,115 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:22,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741960_1136 (size=12001) 2024-12-03T09:14:22,120 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/C of 7bd72df0dd602d0d382d15e837ff9272 into bc8b32bc223e4b7a80dc8bd03a67b703(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:22,122 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:22,122 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/C, priority=13, startTime=1733217261961; duration=0sec 2024-12-03T09:14:22,122 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:22,122 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:C 2024-12-03T09:14:22,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58054 deadline: 1733217322119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:22,127 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:22,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:22,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T09:14:22,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:22,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:22,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:22,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217322140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217322139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:22,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:22,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:22,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T09:14:22,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:22,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:22,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:22,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T09:14:22,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217322349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:22,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:22,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217322350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:22,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:22,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T09:14:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:22,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,500 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/185858289cf3430da4505d7eca519f86 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/185858289cf3430da4505d7eca519f86 2024-12-03T09:14:22,506 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/A of 7bd72df0dd602d0d382d15e837ff9272 into 185858289cf3430da4505d7eca519f86(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:22,506 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:22,506 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/A, priority=13, startTime=1733217261961; duration=0sec 2024-12-03T09:14:22,506 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:22,506 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:A 2024-12-03T09:14:22,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/34698cf1ac0c48948fe68e964e5abde0 2024-12-03T09:14:22,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/c9eb0cd6e3a14c019df625927700128e is 50, key is test_row_0/C:col10/1733217261854/Put/seqid=0 2024-12-03T09:14:22,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741961_1137 (size=12001) 2024-12-03T09:14:22,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T09:14:22,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:22,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:22,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:22,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T09:14:22,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217322659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:22,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217322659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:22,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:22,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:22,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T09:14:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:22,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T09:14:22,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:22,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:22,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:22,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:22,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/c9eb0cd6e3a14c019df625927700128e 2024-12-03T09:14:22,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/fc9125b805fa4ab799fc7163ee750c90 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/fc9125b805fa4ab799fc7163ee750c90 2024-12-03T09:14:22,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/fc9125b805fa4ab799fc7163ee750c90, entries=150, sequenceid=97, filesize=30.2 K 2024-12-03T09:14:22,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/34698cf1ac0c48948fe68e964e5abde0 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/34698cf1ac0c48948fe68e964e5abde0 2024-12-03T09:14:22,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/34698cf1ac0c48948fe68e964e5abde0, entries=150, sequenceid=97, filesize=11.7 K 2024-12-03T09:14:22,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/c9eb0cd6e3a14c019df625927700128e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/c9eb0cd6e3a14c019df625927700128e 2024-12-03T09:14:22,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/c9eb0cd6e3a14c019df625927700128e, entries=150, sequenceid=97, filesize=11.7 K 2024-12-03T09:14:22,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 7bd72df0dd602d0d382d15e837ff9272 in 999ms, sequenceid=97, compaction requested=false 2024-12-03T09:14:22,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:23,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T09:14:23,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:23,055 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:14:23,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:23,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:23,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:23,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:23,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:23,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:23,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120329a302ad2c2a4eea86e28da4dd49a1ab_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217262022/Put/seqid=0 2024-12-03T09:14:23,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741962_1138 (size=12154) 2024-12-03T09:14:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T09:14:23,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:23,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733217263197, max timestamp=1733217263233Num uniques 859; 2024-12-03T09:14:23,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733217263197, max timestamp=1733217263233Num uniques 859; 2024-12-03T09:14:23,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-03T09:14:23,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-03T09:14:23,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733217263197, max timestamp=1733217263233Num uniques 859; 2024-12-03T09:14:23,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-03T09:14:23,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:23,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217323284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217323286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:23,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217323399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:23,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217323400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:23,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:23,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:23,509 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120329a302ad2c2a4eea86e28da4dd49a1ab_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120329a302ad2c2a4eea86e28da4dd49a1ab_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:23,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/35fd1c22cea24115a4e0bcab56537151, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/35fd1c22cea24115a4e0bcab56537151 is 175, key is test_row_0/A:col10/1733217262022/Put/seqid=0 2024-12-03T09:14:23,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741963_1139 (size=30955) 2024-12-03T09:14:23,535 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/35fd1c22cea24115a4e0bcab56537151 2024-12-03T09:14:23,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/387c9c444d7b4ed9a30ca9fa17737f61 is 50, key is test_row_0/B:col10/1733217262022/Put/seqid=0 2024-12-03T09:14:23,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741964_1140 (size=12001) 2024-12-03T09:14:23,560 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/387c9c444d7b4ed9a30ca9fa17737f61 2024-12-03T09:14:23,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/496ce4e3c49e493aaa2c554989a8dd9b is 50, key is test_row_0/C:col10/1733217262022/Put/seqid=0 2024-12-03T09:14:23,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741965_1141 (size=12001) 2024-12-03T09:14:23,604 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/496ce4e3c49e493aaa2c554989a8dd9b 2024-12-03T09:14:23,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217323610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:23,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/35fd1c22cea24115a4e0bcab56537151 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/35fd1c22cea24115a4e0bcab56537151 2024-12-03T09:14:23,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217323610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:23,619 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/35fd1c22cea24115a4e0bcab56537151, entries=150, sequenceid=110, filesize=30.2 K 2024-12-03T09:14:23,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/387c9c444d7b4ed9a30ca9fa17737f61 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/387c9c444d7b4ed9a30ca9fa17737f61 2024-12-03T09:14:23,627 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/387c9c444d7b4ed9a30ca9fa17737f61, entries=150, sequenceid=110, filesize=11.7 K 2024-12-03T09:14:23,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/496ce4e3c49e493aaa2c554989a8dd9b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/496ce4e3c49e493aaa2c554989a8dd9b 2024-12-03T09:14:23,636 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/496ce4e3c49e493aaa2c554989a8dd9b, entries=150, sequenceid=110, filesize=11.7 K 2024-12-03T09:14:23,638 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 7bd72df0dd602d0d382d15e837ff9272 in 584ms, sequenceid=110, compaction requested=true 2024-12-03T09:14:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-03T09:14:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-03T09:14:23,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-03T09:14:23,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6620 sec 2024-12-03T09:14:23,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.6710 sec 2024-12-03T09:14:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:23,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-12-03T09:14:23,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:23,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:23,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:23,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:23,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:23,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:23,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217323951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:23,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217323951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:23,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:23,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033912a14ecaf2405db602de6142b49eab_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217263922/Put/seqid=0 2024-12-03T09:14:23,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741966_1142 (size=12304) 2024-12-03T09:14:23,975 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:23,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033912a14ecaf2405db602de6142b49eab_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033912a14ecaf2405db602de6142b49eab_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:23,982 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/383b03ffb7264985963586195a4da969, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:23,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/383b03ffb7264985963586195a4da969 is 175, key is test_row_0/A:col10/1733217263922/Put/seqid=0 2024-12-03T09:14:24,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741967_1143 (size=31105) 2024-12-03T09:14:24,005 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/383b03ffb7264985963586195a4da969 2024-12-03T09:14:24,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/1ba4b01c8efe425083674e47f8592f85 is 50, key is test_row_0/B:col10/1733217263922/Put/seqid=0 2024-12-03T09:14:24,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741968_1144 (size=12151) 2024-12-03T09:14:24,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/1ba4b01c8efe425083674e47f8592f85 2024-12-03T09:14:24,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/b743d873cd8e4960872f7ba7662934da is 50, key is test_row_0/C:col10/1733217263922/Put/seqid=0 2024-12-03T09:14:24,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217324059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:24,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217324059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:24,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741969_1145 (size=12151) 2024-12-03T09:14:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T09:14:24,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:24,130 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58108 deadline: 1733217324129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:24,130 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:24,130 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-03T09:14:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T09:14:24,134 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:24,135 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:24,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:24,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1733217324139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:24,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58054 deadline: 1733217324140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:24,144 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:24,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:24,144 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:24,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T09:14:24,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217324269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:24,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217324269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:24,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T09:14:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T09:14:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:24,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T09:14:24,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/b743d873cd8e4960872f7ba7662934da 2024-12-03T09:14:24,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/383b03ffb7264985963586195a4da969 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/383b03ffb7264985963586195a4da969 2024-12-03T09:14:24,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/383b03ffb7264985963586195a4da969, entries=150, sequenceid=146, filesize=30.4 K 2024-12-03T09:14:24,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/1ba4b01c8efe425083674e47f8592f85 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/1ba4b01c8efe425083674e47f8592f85 2024-12-03T09:14:24,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/1ba4b01c8efe425083674e47f8592f85, entries=150, sequenceid=146, filesize=11.9 K 2024-12-03T09:14:24,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/b743d873cd8e4960872f7ba7662934da as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/b743d873cd8e4960872f7ba7662934da 2024-12-03T09:14:24,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/b743d873cd8e4960872f7ba7662934da, entries=150, sequenceid=146, filesize=11.9 K 2024-12-03T09:14:24,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=40.25 KB/41220 for 7bd72df0dd602d0d382d15e837ff9272 in 573ms, sequenceid=146, compaction requested=true 2024-12-03T09:14:24,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:24,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:14:24,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:24,496 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:24,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:14:24,496 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:24,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:24,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:14:24,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:24,497 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:24,497 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/B is initiating minor compaction (all files) 2024-12-03T09:14:24,497 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/B in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,498 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/088efacc166a47f484d28645e36c779c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/34698cf1ac0c48948fe68e964e5abde0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/387c9c444d7b4ed9a30ca9fa17737f61, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/1ba4b01c8efe425083674e47f8592f85] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=47.1 K 2024-12-03T09:14:24,498 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124073 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:24,498 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/A is initiating minor compaction (all files) 2024-12-03T09:14:24,498 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/A in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,498 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/185858289cf3430da4505d7eca519f86, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/fc9125b805fa4ab799fc7163ee750c90, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/35fd1c22cea24115a4e0bcab56537151, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/383b03ffb7264985963586195a4da969] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=121.2 K 2024-12-03T09:14:24,498 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,498 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/185858289cf3430da4505d7eca519f86, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/fc9125b805fa4ab799fc7163ee750c90, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/35fd1c22cea24115a4e0bcab56537151, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/383b03ffb7264985963586195a4da969] 2024-12-03T09:14:24,499 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 088efacc166a47f484d28645e36c779c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733217261082 2024-12-03T09:14:24,499 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 34698cf1ac0c48948fe68e964e5abde0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733217261854 2024-12-03T09:14:24,499 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 185858289cf3430da4505d7eca519f86, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733217261082 2024-12-03T09:14:24,501 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 387c9c444d7b4ed9a30ca9fa17737f61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733217261998 2024-12-03T09:14:24,501 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc9125b805fa4ab799fc7163ee750c90, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733217261854 2024-12-03T09:14:24,502 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ba4b01c8efe425083674e47f8592f85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733217263265 2024-12-03T09:14:24,502 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35fd1c22cea24115a4e0bcab56537151, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733217261998 2024-12-03T09:14:24,502 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 383b03ffb7264985963586195a4da969, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733217263265 2024-12-03T09:14:24,514 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:24,517 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203b30adfd9197249aa9b6b4b7a18f32472_7bd72df0dd602d0d382d15e837ff9272 store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:24,518 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#B#compaction#124 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:24,519 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/57e80466db424f0e857025342b3cb53e is 50, key is test_row_0/B:col10/1733217263922/Put/seqid=0 2024-12-03T09:14:24,522 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203b30adfd9197249aa9b6b4b7a18f32472_7bd72df0dd602d0d382d15e837ff9272, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:24,522 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b30adfd9197249aa9b6b4b7a18f32472_7bd72df0dd602d0d382d15e837ff9272 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:24,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741970_1146 (size=12391) 2024-12-03T09:14:24,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741971_1147 (size=4469) 2024-12-03T09:14:24,542 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/57e80466db424f0e857025342b3cb53e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/57e80466db424f0e857025342b3cb53e 2024-12-03T09:14:24,551 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/B of 7bd72df0dd602d0d382d15e837ff9272 into 57e80466db424f0e857025342b3cb53e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:24,551 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:24,551 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/B, priority=12, startTime=1733217264496; duration=0sec 2024-12-03T09:14:24,551 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:24,551 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:B 2024-12-03T09:14:24,551 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:24,553 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:24,553 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/C is initiating minor compaction (all files) 2024-12-03T09:14:24,553 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/C in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,553 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/bc8b32bc223e4b7a80dc8bd03a67b703, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/c9eb0cd6e3a14c019df625927700128e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/496ce4e3c49e493aaa2c554989a8dd9b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/b743d873cd8e4960872f7ba7662934da] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=47.1 K 2024-12-03T09:14:24,554 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting bc8b32bc223e4b7a80dc8bd03a67b703, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733217261082 2024-12-03T09:14:24,554 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting c9eb0cd6e3a14c019df625927700128e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733217261854 2024-12-03T09:14:24,555 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 496ce4e3c49e493aaa2c554989a8dd9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733217261998 2024-12-03T09:14:24,557 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting b743d873cd8e4960872f7ba7662934da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733217263265 2024-12-03T09:14:24,570 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:24,571 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/1b52211c85b64496aaa3ea521a20456b is 50, key is test_row_0/C:col10/1733217263922/Put/seqid=0 2024-12-03T09:14:24,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:14:24,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:24,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:24,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:24,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:24,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:24,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:24,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T09:14:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:24,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741972_1148 (size=12391) 2024-12-03T09:14:24,616 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/1b52211c85b64496aaa3ea521a20456b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/1b52211c85b64496aaa3ea521a20456b 2024-12-03T09:14:24,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203682c8d8b2f2b47c3b4072b09f80127e1_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217263942/Put/seqid=0 2024-12-03T09:14:24,632 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/C of 7bd72df0dd602d0d382d15e837ff9272 into 1b52211c85b64496aaa3ea521a20456b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:24,632 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:24,632 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/C, priority=12, startTime=1733217264496; duration=0sec 2024-12-03T09:14:24,633 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:24,636 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:C 2024-12-03T09:14:24,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217264615, max timestamp=1733217264644Num uniques 859; 2024-12-03T09:14:24,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:24,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217264615, max timestamp=1733217264644Num uniques 859; 2024-12-03T09:14:24,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:24,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217264615, max timestamp=1733217264644Num uniques 859; 2024-12-03T09:14:24,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:24,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741973_1149 (size=14794) 2024-12-03T09:14:24,659 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:24,665 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203682c8d8b2f2b47c3b4072b09f80127e1_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203682c8d8b2f2b47c3b4072b09f80127e1_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:24,667 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/bba43dca7b4541678b1218cb72a4eb98, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:24,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/bba43dca7b4541678b1218cb72a4eb98 is 175, key is test_row_0/A:col10/1733217263942/Put/seqid=0 2024-12-03T09:14:24,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741974_1150 (size=39749) 2024-12-03T09:14:24,708 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/bba43dca7b4541678b1218cb72a4eb98 2024-12-03T09:14:24,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217324709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:24,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217324709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:24,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:24,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/c23e2be9eea54d98924581dd29ccf06b is 50, key is test_row_0/B:col10/1733217263942/Put/seqid=0 2024-12-03T09:14:24,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741975_1151 (size=12151) 2024-12-03T09:14:24,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T09:14:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T09:14:24,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217324819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217324819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:24,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:24,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T09:14:24,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:24,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:24,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:24,945 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#A#compaction#123 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:24,946 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/813937e3888449dbb88d0d68199737ac is 175, key is test_row_0/A:col10/1733217263922/Put/seqid=0 2024-12-03T09:14:24,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741976_1152 (size=31345) 2024-12-03T09:14:24,965 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/813937e3888449dbb88d0d68199737ac as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/813937e3888449dbb88d0d68199737ac 2024-12-03T09:14:24,972 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/A of 7bd72df0dd602d0d382d15e837ff9272 into 813937e3888449dbb88d0d68199737ac(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:24,972 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:24,972 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/A, priority=12, startTime=1733217264496; duration=0sec 2024-12-03T09:14:24,972 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:24,972 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:A 2024-12-03T09:14:25,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217325029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:25,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217325029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:25,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T09:14:25,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:25,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:25,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:25,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:25,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:25,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/c23e2be9eea54d98924581dd29ccf06b 2024-12-03T09:14:25,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/f2971ee93ba94d48938d9041d43bc66f is 50, key is test_row_0/C:col10/1733217263942/Put/seqid=0 2024-12-03T09:14:25,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741977_1153 (size=12151) 2024-12-03T09:14:25,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T09:14:25,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:25,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:25,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:25,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:25,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:25,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T09:14:25,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:25,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217325339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217325339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:25,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T09:14:25,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:25,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:25,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:25,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:25,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:25,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T09:14:25,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:25,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:25,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:25,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:25,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:25,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/f2971ee93ba94d48938d9041d43bc66f 2024-12-03T09:14:25,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/bba43dca7b4541678b1218cb72a4eb98 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/bba43dca7b4541678b1218cb72a4eb98 2024-12-03T09:14:25,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/bba43dca7b4541678b1218cb72a4eb98, entries=200, sequenceid=157, filesize=38.8 K 2024-12-03T09:14:25,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/c23e2be9eea54d98924581dd29ccf06b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/c23e2be9eea54d98924581dd29ccf06b 2024-12-03T09:14:25,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/c23e2be9eea54d98924581dd29ccf06b, entries=150, sequenceid=157, filesize=11.9 K 2024-12-03T09:14:25,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/f2971ee93ba94d48938d9041d43bc66f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/f2971ee93ba94d48938d9041d43bc66f 2024-12-03T09:14:25,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/f2971ee93ba94d48938d9041d43bc66f, entries=150, sequenceid=157, filesize=11.9 K 2024-12-03T09:14:25,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7bd72df0dd602d0d382d15e837ff9272 in 987ms, sequenceid=157, compaction requested=false 2024-12-03T09:14:25,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:25,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-03T09:14:25,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:25,671 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T09:14:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:25,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e2d8ccb51d7b4772b08b78c4eafafb1e_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217264707/Put/seqid=0 2024-12-03T09:14:25,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741978_1154 (size=12304) 2024-12-03T09:14:25,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:25,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217325864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:25,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:25,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217325871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:25,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:25,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217325979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:25,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:25,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217325979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:25,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:25,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:25,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:26,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:26,135 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e2d8ccb51d7b4772b08b78c4eafafb1e_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e2d8ccb51d7b4772b08b78c4eafafb1e_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:26,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/14b25c9ff1d2450eab9e076e4d15e006, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:26,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/14b25c9ff1d2450eab9e076e4d15e006 is 175, key is test_row_0/A:col10/1733217264707/Put/seqid=0 2024-12-03T09:14:26,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741979_1155 (size=31105) 2024-12-03T09:14:26,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217326189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:26,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217326189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:26,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T09:14:26,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217326499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:26,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217326499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:26,545 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/14b25c9ff1d2450eab9e076e4d15e006 2024-12-03T09:14:26,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/9dc199f155584a2fa2d1670701831069 is 50, key is test_row_0/B:col10/1733217264707/Put/seqid=0 2024-12-03T09:14:26,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741980_1156 (size=12151) 2024-12-03T09:14:26,967 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/9dc199f155584a2fa2d1670701831069 2024-12-03T09:14:26,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/b81dc79b8dd14d6ea2dfc225a866bc58 is 50, key is test_row_0/C:col10/1733217264707/Put/seqid=0 2024-12-03T09:14:26,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741981_1157 (size=12151) 2024-12-03T09:14:27,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:27,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217327009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:27,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217327009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:27,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:27,391 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/b81dc79b8dd14d6ea2dfc225a866bc58 2024-12-03T09:14:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/14b25c9ff1d2450eab9e076e4d15e006 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/14b25c9ff1d2450eab9e076e4d15e006 2024-12-03T09:14:27,402 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/14b25c9ff1d2450eab9e076e4d15e006, entries=150, sequenceid=196, filesize=30.4 K 2024-12-03T09:14:27,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/9dc199f155584a2fa2d1670701831069 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/9dc199f155584a2fa2d1670701831069 2024-12-03T09:14:27,407 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/9dc199f155584a2fa2d1670701831069, entries=150, sequenceid=196, filesize=11.9 K 2024-12-03T09:14:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/b81dc79b8dd14d6ea2dfc225a866bc58 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/b81dc79b8dd14d6ea2dfc225a866bc58 2024-12-03T09:14:27,414 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/b81dc79b8dd14d6ea2dfc225a866bc58, entries=150, sequenceid=196, filesize=11.9 K 2024-12-03T09:14:27,415 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 7bd72df0dd602d0d382d15e837ff9272 in 1744ms, sequenceid=196, compaction requested=true 2024-12-03T09:14:27,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:27,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:27,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-03T09:14:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-03T09:14:27,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-03T09:14:27,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2800 sec 2024-12-03T09:14:27,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 3.2890 sec 2024-12-03T09:14:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:28,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T09:14:28,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:28,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:28,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:28,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:28,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:28,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:28,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031e2dbb88a00b46c681a5c04ba031a25b_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217265863/Put/seqid=0 2024-12-03T09:14:28,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741982_1158 (size=14794) 2024-12-03T09:14:28,051 INFO [master/699c9baf3d29:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T09:14:28,051 INFO [master/699c9baf3d29:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T09:14:28,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733217268045, max timestamp=1733217268059Num uniques 859; 2024-12-03T09:14:28,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733217268045, max timestamp=1733217268059Num uniques 859; 2024-12-03T09:14:28,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733217268045, max timestamp=1733217268059Num uniques 859; 2024-12-03T09:14:28,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T09:14:28,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T09:14:28,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T09:14:28,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217328094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:28,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217328097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:28,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:28,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58108 deadline: 1733217328149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:28,153 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:28,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58054 deadline: 1733217328180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:28,181 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:28,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1733217328189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:28,192 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:28,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:28,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217328199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:28,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217328209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T09:14:28,299 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:28,300 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-03T09:14:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T09:14:28,302 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:28,302 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:28,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T09:14:28,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217328409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:28,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217328419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:28,442 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:28,446 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031e2dbb88a00b46c681a5c04ba031a25b_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031e2dbb88a00b46c681a5c04ba031a25b_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:28,447 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/54dc3deee5cc41aa808b1acd5a0a2ee2, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:28,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/54dc3deee5cc41aa808b1acd5a0a2ee2 is 175, key is test_row_0/A:col10/1733217265863/Put/seqid=0 2024-12-03T09:14:28,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741983_1159 (size=39749) 2024-12-03T09:14:28,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T09:14:28,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:28,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:28,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:28,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:28,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:28,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T09:14:28,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:28,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:28,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:28,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:28,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T09:14:28,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217328719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:28,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:28,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217328729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:28,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:28,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T09:14:28,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:28,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:28,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:28,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:28,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:28,852 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/54dc3deee5cc41aa808b1acd5a0a2ee2 2024-12-03T09:14:28,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/2dbd3f95b8e740d39aa35065b2ec9be6 is 50, key is test_row_0/B:col10/1733217265863/Put/seqid=0 2024-12-03T09:14:28,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741984_1160 (size=12151) 2024-12-03T09:14:28,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T09:14:28,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:28,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:28,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:28,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:28,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T09:14:29,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T09:14:29,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:29,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T09:14:29,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:29,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217329229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:29,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:29,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:29,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:29,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217329239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:29,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:29,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:29,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:29,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/2dbd3f95b8e740d39aa35065b2ec9be6 2024-12-03T09:14:29,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/6a023500dad94237bb700301fe0503ff is 50, key is test_row_0/C:col10/1733217265863/Put/seqid=0 2024-12-03T09:14:29,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741985_1161 (size=12151) 2024-12-03T09:14:29,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T09:14:29,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:29,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T09:14:29,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T09:14:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/6a023500dad94237bb700301fe0503ff 2024-12-03T09:14:29,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T09:14:29,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:29,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:29,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/54dc3deee5cc41aa808b1acd5a0a2ee2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/54dc3deee5cc41aa808b1acd5a0a2ee2 2024-12-03T09:14:29,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/54dc3deee5cc41aa808b1acd5a0a2ee2, entries=200, sequenceid=207, filesize=38.8 K 2024-12-03T09:14:29,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/2dbd3f95b8e740d39aa35065b2ec9be6 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/2dbd3f95b8e740d39aa35065b2ec9be6 2024-12-03T09:14:29,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/2dbd3f95b8e740d39aa35065b2ec9be6, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T09:14:29,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/6a023500dad94237bb700301fe0503ff as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/6a023500dad94237bb700301fe0503ff 2024-12-03T09:14:29,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/6a023500dad94237bb700301fe0503ff, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T09:14:29,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 7bd72df0dd602d0d382d15e837ff9272 in 1679ms, sequenceid=207, compaction requested=true 2024-12-03T09:14:29,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:29,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:14:29,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:29,699 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:29,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:14:29,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:29,699 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:29,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:14:29,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:29,701 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141948 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:29,702 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/A is initiating minor compaction (all files) 2024-12-03T09:14:29,702 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:29,702 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/A in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,702 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/B is initiating minor compaction (all files) 2024-12-03T09:14:29,702 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/B in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,702 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/813937e3888449dbb88d0d68199737ac, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/bba43dca7b4541678b1218cb72a4eb98, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/14b25c9ff1d2450eab9e076e4d15e006, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/54dc3deee5cc41aa808b1acd5a0a2ee2] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=138.6 K 2024-12-03T09:14:29,702 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/57e80466db424f0e857025342b3cb53e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/c23e2be9eea54d98924581dd29ccf06b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/9dc199f155584a2fa2d1670701831069, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/2dbd3f95b8e740d39aa35065b2ec9be6] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=47.7 K 2024-12-03T09:14:29,702 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,702 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/813937e3888449dbb88d0d68199737ac, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/bba43dca7b4541678b1218cb72a4eb98, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/14b25c9ff1d2450eab9e076e4d15e006, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/54dc3deee5cc41aa808b1acd5a0a2ee2] 2024-12-03T09:14:29,702 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 57e80466db424f0e857025342b3cb53e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733217263265 2024-12-03T09:14:29,702 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 813937e3888449dbb88d0d68199737ac, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733217263265 2024-12-03T09:14:29,702 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting c23e2be9eea54d98924581dd29ccf06b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733217263942 2024-12-03T09:14:29,703 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting bba43dca7b4541678b1218cb72a4eb98, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733217263942 2024-12-03T09:14:29,703 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dc199f155584a2fa2d1670701831069, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733217264690 2024-12-03T09:14:29,703 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14b25c9ff1d2450eab9e076e4d15e006, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733217264690 2024-12-03T09:14:29,703 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dbd3f95b8e740d39aa35065b2ec9be6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733217265861 2024-12-03T09:14:29,703 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54dc3deee5cc41aa808b1acd5a0a2ee2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733217265861 2024-12-03T09:14:29,712 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:29,713 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:29,713 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/ec55df94ac644c509ef61b2e8d0f1ae9 is 50, key is test_row_0/B:col10/1733217265863/Put/seqid=0 2024-12-03T09:14:29,715 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203c7a5835d9b4c4d6cbf9574739036c17a_7bd72df0dd602d0d382d15e837ff9272 store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:29,718 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203c7a5835d9b4c4d6cbf9574739036c17a_7bd72df0dd602d0d382d15e837ff9272, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:29,718 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c7a5835d9b4c4d6cbf9574739036c17a_7bd72df0dd602d0d382d15e837ff9272 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:29,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741986_1162 (size=12527) 2024-12-03T09:14:29,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741987_1163 (size=4469) 2024-12-03T09:14:29,728 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#A#compaction#136 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:29,729 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/615426d677f0459583a52d43ce8ae97e is 175, key is test_row_0/A:col10/1733217265863/Put/seqid=0 2024-12-03T09:14:29,732 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/ec55df94ac644c509ef61b2e8d0f1ae9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/ec55df94ac644c509ef61b2e8d0f1ae9 2024-12-03T09:14:29,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741988_1164 (size=31481) 2024-12-03T09:14:29,740 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/B of 7bd72df0dd602d0d382d15e837ff9272 into ec55df94ac644c509ef61b2e8d0f1ae9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:29,740 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:29,740 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/B, priority=12, startTime=1733217269699; duration=0sec 2024-12-03T09:14:29,740 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:29,740 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:B 2024-12-03T09:14:29,740 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:29,744 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:29,744 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/C is initiating minor compaction (all files) 2024-12-03T09:14:29,744 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/C in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,744 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/1b52211c85b64496aaa3ea521a20456b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/f2971ee93ba94d48938d9041d43bc66f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/b81dc79b8dd14d6ea2dfc225a866bc58, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/6a023500dad94237bb700301fe0503ff] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=47.7 K 2024-12-03T09:14:29,746 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b52211c85b64496aaa3ea521a20456b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733217263265 2024-12-03T09:14:29,746 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/615426d677f0459583a52d43ce8ae97e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/615426d677f0459583a52d43ce8ae97e 2024-12-03T09:14:29,746 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f2971ee93ba94d48938d9041d43bc66f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733217263942 2024-12-03T09:14:29,747 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting b81dc79b8dd14d6ea2dfc225a866bc58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1733217264690 2024-12-03T09:14:29,747 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a023500dad94237bb700301fe0503ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733217265861 2024-12-03T09:14:29,757 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/A of 7bd72df0dd602d0d382d15e837ff9272 into 615426d677f0459583a52d43ce8ae97e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:29,757 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:29,757 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/A, priority=12, startTime=1733217269699; duration=0sec 2024-12-03T09:14:29,757 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:29,757 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:A 2024-12-03T09:14:29,768 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#C#compaction#137 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:29,768 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/bc61627cf98d46649e0f14e08a001d9f is 50, key is test_row_0/C:col10/1733217265863/Put/seqid=0 2024-12-03T09:14:29,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741989_1165 (size=12527) 2024-12-03T09:14:29,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-03T09:14:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:29,830 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-03T09:14:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:29,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036fa084d765604a89bb90316be24baf57_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217268092/Put/seqid=0 2024-12-03T09:14:29,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741990_1166 (size=12304) 2024-12-03T09:14:29,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:29,861 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036fa084d765604a89bb90316be24baf57_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036fa084d765604a89bb90316be24baf57_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:29,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/8d74fa89f2f6407fb46a5abf9e4b3a68, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:29,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/8d74fa89f2f6407fb46a5abf9e4b3a68 is 175, key is test_row_0/A:col10/1733217268092/Put/seqid=0 2024-12-03T09:14:29,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741991_1167 (size=31105) 2024-12-03T09:14:30,178 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/bc61627cf98d46649e0f14e08a001d9f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/bc61627cf98d46649e0f14e08a001d9f 2024-12-03T09:14:30,185 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/C of 7bd72df0dd602d0d382d15e837ff9272 into bc61627cf98d46649e0f14e08a001d9f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:30,185 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:30,185 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/C, priority=12, startTime=1733217269699; duration=0sec 2024-12-03T09:14:30,186 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:30,186 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:C 2024-12-03T09:14:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:30,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:30,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217330258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:30,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217330259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:30,262 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:30,269 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/8d74fa89f2f6407fb46a5abf9e4b3a68 2024-12-03T09:14:30,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/d7cec446af5a40e78eaa586ade3c2dee is 50, key is test_row_0/B:col10/1733217268092/Put/seqid=0 2024-12-03T09:14:30,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741992_1168 (size=12151) 2024-12-03T09:14:30,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217330369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T09:14:30,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217330579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:30,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:30,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:30,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:30,681 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/d7cec446af5a40e78eaa586ade3c2dee 2024-12-03T09:14:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/5e522115f79f4e65877a208b63e2c2e4 is 50, key is test_row_0/C:col10/1733217268092/Put/seqid=0 2024-12-03T09:14:30,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741993_1169 (size=12151) 2024-12-03T09:14:30,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217330889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:30,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:30,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:30,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:31,094 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/5e522115f79f4e65877a208b63e2c2e4 2024-12-03T09:14:31,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/8d74fa89f2f6407fb46a5abf9e4b3a68 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/8d74fa89f2f6407fb46a5abf9e4b3a68 2024-12-03T09:14:31,104 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/8d74fa89f2f6407fb46a5abf9e4b3a68, entries=150, sequenceid=243, filesize=30.4 K 2024-12-03T09:14:31,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/d7cec446af5a40e78eaa586ade3c2dee as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d7cec446af5a40e78eaa586ade3c2dee 2024-12-03T09:14:31,110 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d7cec446af5a40e78eaa586ade3c2dee, entries=150, sequenceid=243, filesize=11.9 K 2024-12-03T09:14:31,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/5e522115f79f4e65877a208b63e2c2e4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/5e522115f79f4e65877a208b63e2c2e4 2024-12-03T09:14:31,116 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/5e522115f79f4e65877a208b63e2c2e4, entries=150, sequenceid=243, filesize=11.9 K 2024-12-03T09:14:31,117 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 7bd72df0dd602d0d382d15e837ff9272 in 1287ms, sequenceid=243, compaction requested=false 2024-12-03T09:14:31,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:31,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:31,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-03T09:14:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-03T09:14:31,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-03T09:14:31,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8170 sec 2024-12-03T09:14:31,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.8210 sec 2024-12-03T09:14:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:31,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T09:14:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:31,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:31,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039a905eac3897417293bff6cc34c5d15d_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217270257/Put/seqid=0 2024-12-03T09:14:31,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741994_1170 (size=14794) 2024-12-03T09:14:31,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217331458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:31,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:31,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217331569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:31,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:31,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:31,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:31,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217331779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:31,813 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:31,818 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039a905eac3897417293bff6cc34c5d15d_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039a905eac3897417293bff6cc34c5d15d_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:31,820 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/5a9555de673c4f85aa2239b209480a57, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:31,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/5a9555de673c4f85aa2239b209480a57 is 175, key is test_row_0/A:col10/1733217270257/Put/seqid=0 2024-12-03T09:14:31,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741995_1171 (size=39749) 2024-12-03T09:14:31,828 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/5a9555de673c4f85aa2239b209480a57 2024-12-03T09:14:31,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/25b8ed91c22446c9a3bffde832dba615 is 50, key is test_row_0/B:col10/1733217270257/Put/seqid=0 2024-12-03T09:14:31,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741996_1172 (size=12151) 2024-12-03T09:14:32,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:32,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217332089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:32,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/25b8ed91c22446c9a3bffde832dba615 2024-12-03T09:14:32,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/2929794bd0de42069c321854e2a624ba is 50, key is test_row_0/C:col10/1733217270257/Put/seqid=0 2024-12-03T09:14:32,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741997_1173 (size=12151) 2024-12-03T09:14:32,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217332289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:32,291 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T09:14:32,470 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:32,470 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-03T09:14:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T09:14:32,472 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:32,472 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:32,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:32,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T09:14:32,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217332599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:32,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T09:14:32,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:32,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:32,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:32,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:32,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:32,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/2929794bd0de42069c321854e2a624ba 2024-12-03T09:14:32,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/5a9555de673c4f85aa2239b209480a57 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/5a9555de673c4f85aa2239b209480a57 2024-12-03T09:14:32,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/5a9555de673c4f85aa2239b209480a57, entries=200, sequenceid=256, filesize=38.8 K 2024-12-03T09:14:32,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/25b8ed91c22446c9a3bffde832dba615 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/25b8ed91c22446c9a3bffde832dba615 2024-12-03T09:14:32,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/25b8ed91c22446c9a3bffde832dba615, entries=150, sequenceid=256, filesize=11.9 K 2024-12-03T09:14:32,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/2929794bd0de42069c321854e2a624ba as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/2929794bd0de42069c321854e2a624ba 2024-12-03T09:14:32,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/2929794bd0de42069c321854e2a624ba, entries=150, sequenceid=256, filesize=11.9 K 2024-12-03T09:14:32,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 7bd72df0dd602d0d382d15e837ff9272 in 1300ms, sequenceid=256, compaction requested=true 2024-12-03T09:14:32,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:32,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:14:32,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:32,700 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:32,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:14:32,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:32,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:14:32,700 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:32,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:32,701 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:32,702 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/A is initiating minor compaction (all files) 2024-12-03T09:14:32,702 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/A in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:32,702 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/615426d677f0459583a52d43ce8ae97e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/8d74fa89f2f6407fb46a5abf9e4b3a68, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/5a9555de673c4f85aa2239b209480a57] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=99.9 K 2024-12-03T09:14:32,702 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:32,702 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/615426d677f0459583a52d43ce8ae97e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/8d74fa89f2f6407fb46a5abf9e4b3a68, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/5a9555de673c4f85aa2239b209480a57] 2024-12-03T09:14:32,702 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:32,702 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/B is initiating minor compaction (all files) 2024-12-03T09:14:32,703 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/B in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:32,703 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/ec55df94ac644c509ef61b2e8d0f1ae9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d7cec446af5a40e78eaa586ade3c2dee, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/25b8ed91c22446c9a3bffde832dba615] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=36.0 K 2024-12-03T09:14:32,703 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 615426d677f0459583a52d43ce8ae97e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733217265861 2024-12-03T09:14:32,703 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d74fa89f2f6407fb46a5abf9e4b3a68, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733217268082 2024-12-03T09:14:32,703 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting ec55df94ac644c509ef61b2e8d0f1ae9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733217265861 2024-12-03T09:14:32,704 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a9555de673c4f85aa2239b209480a57, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733217270253 2024-12-03T09:14:32,704 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting d7cec446af5a40e78eaa586ade3c2dee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733217268082 2024-12-03T09:14:32,704 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 25b8ed91c22446c9a3bffde832dba615, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733217270255 2024-12-03T09:14:32,714 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#B#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:32,715 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/58b46adf60f54c56a82d301626def78d is 50, key is test_row_0/B:col10/1733217270257/Put/seqid=0 2024-12-03T09:14:32,725 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:32,729 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412031b23556f1b714ac09996e040a3291a29_7bd72df0dd602d0d382d15e837ff9272 store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:32,732 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412031b23556f1b714ac09996e040a3291a29_7bd72df0dd602d0d382d15e837ff9272, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:32,732 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031b23556f1b714ac09996e040a3291a29_7bd72df0dd602d0d382d15e837ff9272 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:32,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741998_1174 (size=12629) 2024-12-03T09:14:32,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741999_1175 (size=4469) 2024-12-03T09:14:32,748 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#A#compaction#145 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:32,748 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/84c18b12e232461cbf9f6be998cbb326 is 175, key is test_row_0/A:col10/1733217270257/Put/seqid=0 2024-12-03T09:14:32,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742000_1176 (size=31583) 2024-12-03T09:14:32,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-03T09:14:32,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:32,778 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T09:14:32,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:32,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:32,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:32,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:32,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:32,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:32,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d8c15a5d5ef44c2a93254f814dd0ca48_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217271451/Put/seqid=0 2024-12-03T09:14:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T09:14:32,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742001_1177 (size=12454) 2024-12-03T09:14:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T09:14:33,147 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/58b46adf60f54c56a82d301626def78d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/58b46adf60f54c56a82d301626def78d 2024-12-03T09:14:33,152 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/B of 7bd72df0dd602d0d382d15e837ff9272 into 58b46adf60f54c56a82d301626def78d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:33,152 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:33,152 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/B, priority=13, startTime=1733217272700; duration=0sec 2024-12-03T09:14:33,153 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:33,153 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:B 2024-12-03T09:14:33,153 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:33,154 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:33,154 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/C is initiating minor compaction (all files) 2024-12-03T09:14:33,154 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/C in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:33,154 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/bc61627cf98d46649e0f14e08a001d9f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/5e522115f79f4e65877a208b63e2c2e4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/2929794bd0de42069c321854e2a624ba] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=36.0 K 2024-12-03T09:14:33,155 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting bc61627cf98d46649e0f14e08a001d9f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733217265861 2024-12-03T09:14:33,155 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e522115f79f4e65877a208b63e2c2e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733217268082 2024-12-03T09:14:33,156 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2929794bd0de42069c321854e2a624ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733217270255 2024-12-03T09:14:33,160 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/84c18b12e232461cbf9f6be998cbb326 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/84c18b12e232461cbf9f6be998cbb326 2024-12-03T09:14:33,167 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/A of 7bd72df0dd602d0d382d15e837ff9272 into 84c18b12e232461cbf9f6be998cbb326(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:33,167 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:33,167 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/A, priority=13, startTime=1733217272700; duration=0sec 2024-12-03T09:14:33,167 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:33,167 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:A 2024-12-03T09:14:33,167 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#C#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:33,168 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/854346728a334d4e9b595d76868f7a30 is 50, key is test_row_0/C:col10/1733217270257/Put/seqid=0 2024-12-03T09:14:33,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742002_1178 (size=12629) 2024-12-03T09:14:33,188 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/854346728a334d4e9b595d76868f7a30 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/854346728a334d4e9b595d76868f7a30 2024-12-03T09:14:33,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:33,194 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d8c15a5d5ef44c2a93254f814dd0ca48_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d8c15a5d5ef44c2a93254f814dd0ca48_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:33,195 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/C of 7bd72df0dd602d0d382d15e837ff9272 into 854346728a334d4e9b595d76868f7a30(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:33,195 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:33,195 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/C, priority=13, startTime=1733217272700; duration=0sec 2024-12-03T09:14:33,195 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:33,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/ab3a794ff5a442fca94cf6ac112b5b0d, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:33,195 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:C 2024-12-03T09:14:33,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/ab3a794ff5a442fca94cf6ac112b5b0d is 175, key is test_row_0/A:col10/1733217271451/Put/seqid=0 2024-12-03T09:14:33,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742003_1179 (size=31255) 2024-12-03T09:14:33,600 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/ab3a794ff5a442fca94cf6ac112b5b0d 2024-12-03T09:14:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T09:14:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/27675729a6354481a8346692b1ccbd9c is 50, key is test_row_0/B:col10/1733217271451/Put/seqid=0 2024-12-03T09:14:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:33,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:33,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742004_1180 (size=12301) 2024-12-03T09:14:33,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217333644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:33,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:33,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:33,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217333749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:33,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:33,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217333959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:33,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:34,018 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/27675729a6354481a8346692b1ccbd9c 2024-12-03T09:14:34,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/709a4f06cfe5477dbcda269725c48dcb is 50, key is test_row_0/C:col10/1733217271451/Put/seqid=0 2024-12-03T09:14:34,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742005_1181 (size=12301) 2024-12-03T09:14:34,030 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/709a4f06cfe5477dbcda269725c48dcb 2024-12-03T09:14:34,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/ab3a794ff5a442fca94cf6ac112b5b0d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/ab3a794ff5a442fca94cf6ac112b5b0d 2024-12-03T09:14:34,039 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/ab3a794ff5a442fca94cf6ac112b5b0d, entries=150, sequenceid=276, filesize=30.5 K 2024-12-03T09:14:34,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/27675729a6354481a8346692b1ccbd9c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/27675729a6354481a8346692b1ccbd9c 2024-12-03T09:14:34,045 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/27675729a6354481a8346692b1ccbd9c, entries=150, sequenceid=276, filesize=12.0 K 2024-12-03T09:14:34,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/709a4f06cfe5477dbcda269725c48dcb as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/709a4f06cfe5477dbcda269725c48dcb 2024-12-03T09:14:34,051 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/709a4f06cfe5477dbcda269725c48dcb, entries=150, sequenceid=276, filesize=12.0 K 2024-12-03T09:14:34,052 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 7bd72df0dd602d0d382d15e837ff9272 in 1275ms, sequenceid=276, compaction requested=false 2024-12-03T09:14:34,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:34,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:34,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-03T09:14:34,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-03T09:14:34,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-03T09:14:34,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5810 sec 2024-12-03T09:14:34,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.5870 sec 2024-12-03T09:14:34,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:34,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T09:14:34,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:34,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:34,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:34,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:34,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:34,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:34,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b6717a1b62a84e739993641a08961888_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217273640/Put/seqid=0 2024-12-03T09:14:34,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742006_1182 (size=14994) 2024-12-03T09:14:34,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217334320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:34,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217334430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:34,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:34,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:34,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T09:14:34,619 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:34,620 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-03T09:14:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T09:14:34,622 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:34,622 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:34,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:34,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:34,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217334639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:34,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:34,682 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:34,686 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b6717a1b62a84e739993641a08961888_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b6717a1b62a84e739993641a08961888_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:34,687 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/be67871c998443a1b29897849b07cd9d, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:34,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/be67871c998443a1b29897849b07cd9d is 175, key is test_row_0/A:col10/1733217273640/Put/seqid=0 2024-12-03T09:14:34,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742007_1183 (size=39949) 2024-12-03T09:14:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T09:14:34,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T09:14:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:34,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:34,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T09:14:34,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:34,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:34,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:34,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:34,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T09:14:34,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217334949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:34,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:34,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:34,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:35,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T09:14:35,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:35,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:35,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:35,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:35,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:35,097 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/be67871c998443a1b29897849b07cd9d 2024-12-03T09:14:35,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/62019d2f4a3348c7bdff81c58455459c is 50, key is test_row_0/B:col10/1733217273640/Put/seqid=0 2024-12-03T09:14:35,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742008_1184 (size=12301) 2024-12-03T09:14:35,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T09:14:35,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:35,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:35,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:35,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:35,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:35,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T09:14:35,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T09:14:35,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:35,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:35,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:35,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:35,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:35,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217335459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:35,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/62019d2f4a3348c7bdff81c58455459c 2024-12-03T09:14:35,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/885f64175a3344c29460c7b8a0b9ce69 is 50, key is test_row_0/C:col10/1733217273640/Put/seqid=0 2024-12-03T09:14:35,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742009_1185 (size=12301) 2024-12-03T09:14:35,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/885f64175a3344c29460c7b8a0b9ce69 2024-12-03T09:14:35,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/be67871c998443a1b29897849b07cd9d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/be67871c998443a1b29897849b07cd9d 2024-12-03T09:14:35,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/be67871c998443a1b29897849b07cd9d, entries=200, sequenceid=292, filesize=39.0 K 2024-12-03T09:14:35,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/62019d2f4a3348c7bdff81c58455459c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/62019d2f4a3348c7bdff81c58455459c 2024-12-03T09:14:35,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/62019d2f4a3348c7bdff81c58455459c, entries=150, sequenceid=292, filesize=12.0 K 2024-12-03T09:14:35,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/885f64175a3344c29460c7b8a0b9ce69 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/885f64175a3344c29460c7b8a0b9ce69 2024-12-03T09:14:35,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T09:14:35,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:35,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:35,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:35,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:35,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:35,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/885f64175a3344c29460c7b8a0b9ce69, entries=150, sequenceid=292, filesize=12.0 K 2024-12-03T09:14:35,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7bd72df0dd602d0d382d15e837ff9272 in 1275ms, sequenceid=292, compaction requested=true 2024-12-03T09:14:35,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:35,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:14:35,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:35,547 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:35,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:14:35,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:35,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:14:35,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T09:14:35,547 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:35,548 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:35,548 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:35,548 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/B is initiating minor compaction (all files) 2024-12-03T09:14:35,548 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/A is initiating minor compaction (all files) 2024-12-03T09:14:35,548 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/B in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:35,548 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/A in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:35,549 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/84c18b12e232461cbf9f6be998cbb326, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/ab3a794ff5a442fca94cf6ac112b5b0d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/be67871c998443a1b29897849b07cd9d] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=100.4 K 2024-12-03T09:14:35,549 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/58b46adf60f54c56a82d301626def78d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/27675729a6354481a8346692b1ccbd9c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/62019d2f4a3348c7bdff81c58455459c] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=36.4 K 2024-12-03T09:14:35,549 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:35,549 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/84c18b12e232461cbf9f6be998cbb326, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/ab3a794ff5a442fca94cf6ac112b5b0d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/be67871c998443a1b29897849b07cd9d] 2024-12-03T09:14:35,549 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 58b46adf60f54c56a82d301626def78d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733217270255 2024-12-03T09:14:35,549 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84c18b12e232461cbf9f6be998cbb326, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733217270255 2024-12-03T09:14:35,549 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 27675729a6354481a8346692b1ccbd9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733217271451 2024-12-03T09:14:35,550 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab3a794ff5a442fca94cf6ac112b5b0d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733217271451 2024-12-03T09:14:35,550 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 62019d2f4a3348c7bdff81c58455459c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733217273640 2024-12-03T09:14:35,550 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting be67871c998443a1b29897849b07cd9d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733217273633 2024-12-03T09:14:35,562 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:35,565 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#B#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:35,565 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/36ed7556fb4e402f9764d5e0a0c670bc is 50, key is test_row_0/B:col10/1733217273640/Put/seqid=0 2024-12-03T09:14:35,568 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412037c349034c3194b33b5d8a334fb5a181c_7bd72df0dd602d0d382d15e837ff9272 store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:35,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742010_1186 (size=12881) 2024-12-03T09:14:35,571 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412037c349034c3194b33b5d8a334fb5a181c_7bd72df0dd602d0d382d15e837ff9272, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:35,572 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037c349034c3194b33b5d8a334fb5a181c_7bd72df0dd602d0d382d15e837ff9272 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:35,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742011_1187 (size=4469) 2024-12-03T09:14:35,581 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/36ed7556fb4e402f9764d5e0a0c670bc as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/36ed7556fb4e402f9764d5e0a0c670bc 2024-12-03T09:14:35,582 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#A#compaction#153 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:35,582 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/4b89e46d22734cff9bfd6df0a80ec120 is 175, key is test_row_0/A:col10/1733217273640/Put/seqid=0 2024-12-03T09:14:35,587 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/B of 7bd72df0dd602d0d382d15e837ff9272 into 36ed7556fb4e402f9764d5e0a0c670bc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:35,587 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:35,588 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/B, priority=13, startTime=1733217275547; duration=0sec 2024-12-03T09:14:35,588 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:35,588 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:B 2024-12-03T09:14:35,588 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:35,589 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:35,589 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/C is initiating minor compaction (all files) 2024-12-03T09:14:35,589 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/C in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:35,589 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/854346728a334d4e9b595d76868f7a30, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/709a4f06cfe5477dbcda269725c48dcb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/885f64175a3344c29460c7b8a0b9ce69] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=36.4 K 2024-12-03T09:14:35,590 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 854346728a334d4e9b595d76868f7a30, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733217270255 2024-12-03T09:14:35,590 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 709a4f06cfe5477dbcda269725c48dcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733217271451 2024-12-03T09:14:35,590 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 885f64175a3344c29460c7b8a0b9ce69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733217273640 2024-12-03T09:14:35,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742012_1188 (size=31835) 2024-12-03T09:14:35,602 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:35,603 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/30c1b33645d144da8d9b92f2a5c5b632 is 50, key is test_row_0/C:col10/1733217273640/Put/seqid=0 2024-12-03T09:14:35,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742013_1189 (size=12881) 2024-12-03T09:14:35,614 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/30c1b33645d144da8d9b92f2a5c5b632 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/30c1b33645d144da8d9b92f2a5c5b632 2024-12-03T09:14:35,621 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/C of 7bd72df0dd602d0d382d15e837ff9272 into 30c1b33645d144da8d9b92f2a5c5b632(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:35,621 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:35,621 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/C, priority=13, startTime=1733217275547; duration=0sec 2024-12-03T09:14:35,621 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:35,621 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:C 2024-12-03T09:14:35,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-03T09:14:35,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:35,694 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T09:14:35,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:35,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:35,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:35,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:35,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:35,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:35,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035f0d7b2ec07045c1afd1dba50f67b378_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217274307/Put/seqid=0 2024-12-03T09:14:35,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742014_1190 (size=12454) 2024-12-03T09:14:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T09:14:36,003 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/4b89e46d22734cff9bfd6df0a80ec120 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/4b89e46d22734cff9bfd6df0a80ec120 2024-12-03T09:14:36,007 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/A of 7bd72df0dd602d0d382d15e837ff9272 into 4b89e46d22734cff9bfd6df0a80ec120(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:36,008 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:36,008 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/A, priority=13, startTime=1733217275547; duration=0sec 2024-12-03T09:14:36,008 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:36,008 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:A 2024-12-03T09:14:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:36,128 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035f0d7b2ec07045c1afd1dba50f67b378_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035f0d7b2ec07045c1afd1dba50f67b378_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:36,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/e32afb18b32449d08f803c830c00a857, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:36,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/e32afb18b32449d08f803c830c00a857 is 175, key is test_row_0/A:col10/1733217274307/Put/seqid=0 2024-12-03T09:14:36,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742015_1191 (size=31255) 2024-12-03T09:14:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:36,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:36,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217336350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:36,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217336459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:36,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217336469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:36,471 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:36,539 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/e32afb18b32449d08f803c830c00a857 2024-12-03T09:14:36,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/6d93d3e930da486ba836b5dadb97e8a1 is 50, key is test_row_0/B:col10/1733217274307/Put/seqid=0 2024-12-03T09:14:36,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742016_1192 (size=12301) 2024-12-03T09:14:36,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217336669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T09:14:36,955 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/6d93d3e930da486ba836b5dadb97e8a1 2024-12-03T09:14:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/1f9875a6e45f4bf19925d8007cf2e13f is 50, key is test_row_0/C:col10/1733217274307/Put/seqid=0 2024-12-03T09:14:36,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742017_1193 (size=12301) 2024-12-03T09:14:36,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:36,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217336979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:36,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:36,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:36,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:37,371 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/1f9875a6e45f4bf19925d8007cf2e13f 2024-12-03T09:14:37,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/e32afb18b32449d08f803c830c00a857 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/e32afb18b32449d08f803c830c00a857 2024-12-03T09:14:37,382 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/e32afb18b32449d08f803c830c00a857, entries=150, sequenceid=313, filesize=30.5 K 2024-12-03T09:14:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/6d93d3e930da486ba836b5dadb97e8a1 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/6d93d3e930da486ba836b5dadb97e8a1 2024-12-03T09:14:37,395 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/6d93d3e930da486ba836b5dadb97e8a1, entries=150, sequenceid=313, filesize=12.0 K 2024-12-03T09:14:37,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/1f9875a6e45f4bf19925d8007cf2e13f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/1f9875a6e45f4bf19925d8007cf2e13f 2024-12-03T09:14:37,401 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/1f9875a6e45f4bf19925d8007cf2e13f, entries=150, sequenceid=313, filesize=12.0 K 2024-12-03T09:14:37,403 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7bd72df0dd602d0d382d15e837ff9272 in 1709ms, sequenceid=313, compaction requested=false 2024-12-03T09:14:37,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:37,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:37,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-03T09:14:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-03T09:14:37,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-03T09:14:37,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7820 sec 2024-12-03T09:14:37,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.7880 sec 2024-12-03T09:14:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:37,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T09:14:37,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:37,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:37,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:37,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:37,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:37,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:37,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203524fc152bdfc4401acdb8abef6980354_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217276349/Put/seqid=0 2024-12-03T09:14:37,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742018_1194 (size=14994) 2024-12-03T09:14:37,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217337557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:37,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217337671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:37,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217337880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:37,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:37,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:37,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:37,905 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:37,909 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203524fc152bdfc4401acdb8abef6980354_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203524fc152bdfc4401acdb8abef6980354_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:37,910 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/1dae3cf2ac56468cae58f1b1dd970678, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:37,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/1dae3cf2ac56468cae58f1b1dd970678 is 175, key is test_row_0/A:col10/1733217276349/Put/seqid=0 2024-12-03T09:14:37,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742019_1195 (size=39949) 2024-12-03T09:14:37,916 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/1dae3cf2ac56468cae58f1b1dd970678 2024-12-03T09:14:37,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/eb2b9e67f75f461984b64048071f205a is 50, key is test_row_0/B:col10/1733217276349/Put/seqid=0 2024-12-03T09:14:37,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742020_1196 (size=12301) 2024-12-03T09:14:38,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58108 deadline: 1733217338169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:38,170 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:38,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:38,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217338212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:38,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:38,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58054 deadline: 1733217338290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:38,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:38,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58126 deadline: 1733217338290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:38,291 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18351 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:38,292 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18358 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:38,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/eb2b9e67f75f461984b64048071f205a 2024-12-03T09:14:38,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/173440c98c454b8c81cd861471ab2209 is 50, key is test_row_0/C:col10/1733217276349/Put/seqid=0 2024-12-03T09:14:38,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742021_1197 (size=12301) 2024-12-03T09:14:38,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/173440c98c454b8c81cd861471ab2209 2024-12-03T09:14:38,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/1dae3cf2ac56468cae58f1b1dd970678 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/1dae3cf2ac56468cae58f1b1dd970678 2024-12-03T09:14:38,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/1dae3cf2ac56468cae58f1b1dd970678, entries=200, sequenceid=328, filesize=39.0 K 2024-12-03T09:14:38,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/eb2b9e67f75f461984b64048071f205a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/eb2b9e67f75f461984b64048071f205a 2024-12-03T09:14:38,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/eb2b9e67f75f461984b64048071f205a, entries=150, sequenceid=328, filesize=12.0 K 2024-12-03T09:14:38,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/173440c98c454b8c81cd861471ab2209 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/173440c98c454b8c81cd861471ab2209 2024-12-03T09:14:38,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/173440c98c454b8c81cd861471ab2209, entries=150, sequenceid=328, filesize=12.0 K 2024-12-03T09:14:38,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 7bd72df0dd602d0d382d15e837ff9272 in 931ms, sequenceid=328, compaction requested=true 2024-12-03T09:14:38,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:38,423 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:38,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:14:38,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:38,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:14:38,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:38,423 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:38,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:14:38,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:38,424 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:38,424 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/A is initiating minor compaction (all files) 2024-12-03T09:14:38,424 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/A in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:38,424 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/4b89e46d22734cff9bfd6df0a80ec120, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/e32afb18b32449d08f803c830c00a857, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/1dae3cf2ac56468cae58f1b1dd970678] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=100.6 K 2024-12-03T09:14:38,424 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:38,424 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/4b89e46d22734cff9bfd6df0a80ec120, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/e32afb18b32449d08f803c830c00a857, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/1dae3cf2ac56468cae58f1b1dd970678] 2024-12-03T09:14:38,424 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:38,424 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/B is initiating minor compaction (all files) 2024-12-03T09:14:38,424 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/B in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:38,425 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/36ed7556fb4e402f9764d5e0a0c670bc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/6d93d3e930da486ba836b5dadb97e8a1, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/eb2b9e67f75f461984b64048071f205a] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=36.6 K 2024-12-03T09:14:38,425 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b89e46d22734cff9bfd6df0a80ec120, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733217273640 2024-12-03T09:14:38,425 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ed7556fb4e402f9764d5e0a0c670bc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733217273640 2024-12-03T09:14:38,425 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e32afb18b32449d08f803c830c00a857, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733217274307 2024-12-03T09:14:38,425 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d93d3e930da486ba836b5dadb97e8a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733217274307 2024-12-03T09:14:38,425 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dae3cf2ac56468cae58f1b1dd970678, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733217276332 2024-12-03T09:14:38,425 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting eb2b9e67f75f461984b64048071f205a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733217276332 2024-12-03T09:14:38,434 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:38,437 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#B#compaction#163 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:38,438 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/4013c47f55ff492e961cd8a3d01ad0a2 is 50, key is test_row_0/B:col10/1733217276349/Put/seqid=0 2024-12-03T09:14:38,441 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120395b66ff7999b46a0aa2f54932219fde9_7bd72df0dd602d0d382d15e837ff9272 store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:38,442 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120395b66ff7999b46a0aa2f54932219fde9_7bd72df0dd602d0d382d15e837ff9272, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:38,443 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120395b66ff7999b46a0aa2f54932219fde9_7bd72df0dd602d0d382d15e837ff9272 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:38,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742022_1198 (size=12983) 2024-12-03T09:14:38,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742023_1199 (size=4469) 2024-12-03T09:14:38,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:38,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T09:14:38,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:38,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:38,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:38,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:38,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:38,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:38,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203833c3be873d043e1b3e5f188d157df03_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217277533/Put/seqid=0 2024-12-03T09:14:38,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742024_1200 (size=14994) 2024-12-03T09:14:38,514 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:38,517 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203833c3be873d043e1b3e5f188d157df03_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203833c3be873d043e1b3e5f188d157df03_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:38,518 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/f6430f4911d74555870c50b5eb357bca, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:38,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/f6430f4911d74555870c50b5eb357bca is 175, key is test_row_0/A:col10/1733217277533/Put/seqid=0 2024-12-03T09:14:38,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742025_1201 (size=39949) 2024-12-03T09:14:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217338544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:38,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217338659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:38,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217338729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T09:14:38,779 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:38,780 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-03T09:14:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T09:14:38,781 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:38,782 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:38,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:38,851 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/4013c47f55ff492e961cd8a3d01ad0a2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/4013c47f55ff492e961cd8a3d01ad0a2 2024-12-03T09:14:38,852 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#A#compaction#162 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:38,852 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/2701d807f7dc45ddafc93fd38be59e2a is 175, key is test_row_0/A:col10/1733217276349/Put/seqid=0 2024-12-03T09:14:38,856 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/B of 7bd72df0dd602d0d382d15e837ff9272 into 4013c47f55ff492e961cd8a3d01ad0a2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:38,856 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:38,856 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/B, priority=13, startTime=1733217278423; duration=0sec 2024-12-03T09:14:38,857 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:38,857 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:B 2024-12-03T09:14:38,857 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:38,858 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:38,858 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/C is initiating minor compaction (all files) 2024-12-03T09:14:38,858 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/C in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:38,858 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/30c1b33645d144da8d9b92f2a5c5b632, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/1f9875a6e45f4bf19925d8007cf2e13f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/173440c98c454b8c81cd861471ab2209] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=36.6 K 2024-12-03T09:14:38,859 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 30c1b33645d144da8d9b92f2a5c5b632, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733217273640 2024-12-03T09:14:38,859 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f9875a6e45f4bf19925d8007cf2e13f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733217274307 2024-12-03T09:14:38,859 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 173440c98c454b8c81cd861471ab2209, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733217276332 2024-12-03T09:14:38,866 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:38,867 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/8f5fe620ec484394b4d0f408b1c10ec9 is 50, key is test_row_0/C:col10/1733217276349/Put/seqid=0 2024-12-03T09:14:38,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742026_1202 (size=31937) 2024-12-03T09:14:38,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:38,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217338869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:38,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:38,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:38,877 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/2701d807f7dc45ddafc93fd38be59e2a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/2701d807f7dc45ddafc93fd38be59e2a 2024-12-03T09:14:38,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742027_1203 (size=12983) 2024-12-03T09:14:38,883 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/A of 7bd72df0dd602d0d382d15e837ff9272 into 2701d807f7dc45ddafc93fd38be59e2a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:38,883 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:38,883 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/A, priority=13, startTime=1733217278423; duration=0sec 2024-12-03T09:14:38,883 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:38,883 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:A 2024-12-03T09:14:38,886 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/8f5fe620ec484394b4d0f408b1c10ec9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/8f5fe620ec484394b4d0f408b1c10ec9 2024-12-03T09:14:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T09:14:38,891 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/C of 7bd72df0dd602d0d382d15e837ff9272 into 8f5fe620ec484394b4d0f408b1c10ec9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:38,891 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:38,891 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/C, priority=13, startTime=1733217278423; duration=0sec 2024-12-03T09:14:38,891 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:38,891 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:C 2024-12-03T09:14:38,926 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/f6430f4911d74555870c50b5eb357bca 2024-12-03T09:14:38,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-03T09:14:38,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:38,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:38,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:38,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:38,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:38,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/45cf5b31a5fb417a99969011797eb05a is 50, key is test_row_0/B:col10/1733217277533/Put/seqid=0 2024-12-03T09:14:38,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:38,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742028_1204 (size=12301) 2024-12-03T09:14:38,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/45cf5b31a5fb417a99969011797eb05a 2024-12-03T09:14:38,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/0137905c70074738a76d241a7cf9e3a7 is 50, key is test_row_0/C:col10/1733217277533/Put/seqid=0 2024-12-03T09:14:38,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742029_1205 (size=12301) 2024-12-03T09:14:38,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/0137905c70074738a76d241a7cf9e3a7 2024-12-03T09:14:38,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/f6430f4911d74555870c50b5eb357bca as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/f6430f4911d74555870c50b5eb357bca 2024-12-03T09:14:38,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/f6430f4911d74555870c50b5eb357bca, entries=200, sequenceid=347, filesize=39.0 K 2024-12-03T09:14:38,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/45cf5b31a5fb417a99969011797eb05a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/45cf5b31a5fb417a99969011797eb05a 2024-12-03T09:14:38,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/45cf5b31a5fb417a99969011797eb05a, entries=150, sequenceid=347, filesize=12.0 K 2024-12-03T09:14:38,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/0137905c70074738a76d241a7cf9e3a7 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/0137905c70074738a76d241a7cf9e3a7 2024-12-03T09:14:38,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/0137905c70074738a76d241a7cf9e3a7, entries=150, sequenceid=347, filesize=12.0 K 2024-12-03T09:14:38,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7bd72df0dd602d0d382d15e837ff9272 in 479ms, sequenceid=347, compaction requested=false 2024-12-03T09:14:38,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:39,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-03T09:14:39,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:39,089 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T09:14:39,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:39,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:39,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:39,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:39,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:39,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:39,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034763c4834c7e45158640e087f2d50f3c_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217278526/Put/seqid=0 2024-12-03T09:14:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T09:14:39,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742030_1206 (size=12454) 2024-12-03T09:14:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:39,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. as already flushing 2024-12-03T09:14:39,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217339228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:39,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217339339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:39,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:39,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:39,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T09:14:39,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:39,505 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034763c4834c7e45158640e087f2d50f3c_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034763c4834c7e45158640e087f2d50f3c_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:39,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3c1ba9acba7d4e4093233678135b367d, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:39,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3c1ba9acba7d4e4093233678135b367d is 175, key is test_row_0/A:col10/1733217278526/Put/seqid=0 2024-12-03T09:14:39,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742031_1207 (size=31255) 2024-12-03T09:14:39,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217339549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:39,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58056 deadline: 1733217339750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:39,752 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 7bd72df0dd602d0d382d15e837ff9272 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:39,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:39,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:39,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:39,815 INFO [Thread-616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-616. 2024-12-03T09:14:39,815 DEBUG [Thread-616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:39,815 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:39,815 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:39,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:39,815 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-12-03T09:14:39,815 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:39,815 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:39,815 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:39,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:39,817 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-12-03T09:14:39,817 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:39,817 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:39,817 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:39,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:39,820 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-12-03T09:14:39,820 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:39,820 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:39,821 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:39,821 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:39,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217339860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:39,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:39,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:39,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T09:14:39,921 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3c1ba9acba7d4e4093233678135b367d 2024-12-03T09:14:39,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/e91449bf95d340c08ea998d6c4e742aa is 50, key is test_row_0/B:col10/1733217278526/Put/seqid=0 2024-12-03T09:14:39,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742032_1208 (size=12301) 2024-12-03T09:14:40,339 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/e91449bf95d340c08ea998d6c4e742aa 2024-12-03T09:14:40,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/475db233ba004bcaa0432c5592b382a7 is 50, key is test_row_0/C:col10/1733217278526/Put/seqid=0 2024-12-03T09:14:40,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742033_1209 (size=12301) 2024-12-03T09:14:40,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58114 deadline: 1733217340369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:40,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:14:40,759 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/475db233ba004bcaa0432c5592b382a7 2024-12-03T09:14:40,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3c1ba9acba7d4e4093233678135b367d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3c1ba9acba7d4e4093233678135b367d 2024-12-03T09:14:40,772 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3c1ba9acba7d4e4093233678135b367d, entries=150, sequenceid=363, filesize=30.5 K 2024-12-03T09:14:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/e91449bf95d340c08ea998d6c4e742aa as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/e91449bf95d340c08ea998d6c4e742aa 2024-12-03T09:14:40,778 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/e91449bf95d340c08ea998d6c4e742aa, entries=150, sequenceid=363, filesize=12.0 K 2024-12-03T09:14:40,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/475db233ba004bcaa0432c5592b382a7 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/475db233ba004bcaa0432c5592b382a7 2024-12-03T09:14:40,783 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/475db233ba004bcaa0432c5592b382a7, entries=150, sequenceid=363, filesize=12.0 K 2024-12-03T09:14:40,784 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7bd72df0dd602d0d382d15e837ff9272 in 1695ms, sequenceid=363, compaction requested=true 2024-12-03T09:14:40,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:40,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:40,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-03T09:14:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-03T09:14:40,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-03T09:14:40,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0040 sec 2024-12-03T09:14:40,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.0080 sec 2024-12-03T09:14:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T09:14:40,931 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:41,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T09:14:41,385 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-12-03T09:14:41,385 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:41,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:41,385 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:41,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:41,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:41,386 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:41,386 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:41,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:41,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:41,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:41,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ced10ce0f0534a83b35be4e3e979bb2f_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217281381/Put/seqid=0 2024-12-03T09:14:41,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742034_1210 (size=12454) 2024-12-03T09:14:41,784 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-12-03T09:14:41,785 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:41,785 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:41,785 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:41,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:41,800 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:41,808 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ced10ce0f0534a83b35be4e3e979bb2f_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ced10ce0f0534a83b35be4e3e979bb2f_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:41,810 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3890c994b32148d1aa6cc749da4ec41e, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:41,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3890c994b32148d1aa6cc749da4ec41e is 175, key is test_row_0/A:col10/1733217281381/Put/seqid=0 2024-12-03T09:14:41,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742035_1211 (size=31255) 2024-12-03T09:14:42,218 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3890c994b32148d1aa6cc749da4ec41e 2024-12-03T09:14:42,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/d756696c174c4effaa71d08463b479b7 is 50, key is test_row_0/B:col10/1733217281381/Put/seqid=0 2024-12-03T09:14:42,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742036_1212 (size=12301) 2024-12-03T09:14:42,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/d756696c174c4effaa71d08463b479b7 2024-12-03T09:14:42,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/6449690d3415498794e45c71cb892d36 is 50, key is test_row_0/C:col10/1733217281381/Put/seqid=0 2024-12-03T09:14:42,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742037_1213 (size=12301) 2024-12-03T09:14:42,799 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T09:14:43,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/6449690d3415498794e45c71cb892d36 2024-12-03T09:14:43,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3890c994b32148d1aa6cc749da4ec41e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3890c994b32148d1aa6cc749da4ec41e 2024-12-03T09:14:43,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3890c994b32148d1aa6cc749da4ec41e, entries=150, sequenceid=383, filesize=30.5 K 2024-12-03T09:14:43,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/d756696c174c4effaa71d08463b479b7 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d756696c174c4effaa71d08463b479b7 2024-12-03T09:14:43,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d756696c174c4effaa71d08463b479b7, entries=150, sequenceid=383, filesize=12.0 K 2024-12-03T09:14:43,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/6449690d3415498794e45c71cb892d36 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/6449690d3415498794e45c71cb892d36 2024-12-03T09:14:43,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/6449690d3415498794e45c71cb892d36, entries=150, sequenceid=383, filesize=12.0 K 2024-12-03T09:14:43,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for 7bd72df0dd602d0d382d15e837ff9272 in 1753ms, sequenceid=383, compaction requested=true 2024-12-03T09:14:43,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:43,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:14:43,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:43,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:14:43,138 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:43,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:43,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7bd72df0dd602d0d382d15e837ff9272:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:14:43,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:43,138 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:43,139 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:43,139 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:43,139 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/A is initiating minor compaction (all files) 2024-12-03T09:14:43,139 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/B is initiating minor compaction (all files) 2024-12-03T09:14:43,139 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/B in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:43,139 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/A in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:43,139 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/2701d807f7dc45ddafc93fd38be59e2a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/f6430f4911d74555870c50b5eb357bca, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3c1ba9acba7d4e4093233678135b367d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3890c994b32148d1aa6cc749da4ec41e] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=131.2 K 2024-12-03T09:14:43,139 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/4013c47f55ff492e961cd8a3d01ad0a2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/45cf5b31a5fb417a99969011797eb05a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/e91449bf95d340c08ea998d6c4e742aa, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d756696c174c4effaa71d08463b479b7] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=48.7 K 2024-12-03T09:14:43,139 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:43,139 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/2701d807f7dc45ddafc93fd38be59e2a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/f6430f4911d74555870c50b5eb357bca, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3c1ba9acba7d4e4093233678135b367d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3890c994b32148d1aa6cc749da4ec41e] 2024-12-03T09:14:43,139 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4013c47f55ff492e961cd8a3d01ad0a2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733217276332 2024-12-03T09:14:43,139 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2701d807f7dc45ddafc93fd38be59e2a, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733217276332 2024-12-03T09:14:43,140 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 45cf5b31a5fb417a99969011797eb05a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733217277533 2024-12-03T09:14:43,140 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6430f4911d74555870c50b5eb357bca, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733217277533 2024-12-03T09:14:43,140 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting e91449bf95d340c08ea998d6c4e742aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733217278524 2024-12-03T09:14:43,140 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c1ba9acba7d4e4093233678135b367d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733217278524 2024-12-03T09:14:43,140 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting d756696c174c4effaa71d08463b479b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733217279214 2024-12-03T09:14:43,140 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3890c994b32148d1aa6cc749da4ec41e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733217279214 2024-12-03T09:14:43,154 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#B#compaction#174 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:43,155 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/d2e77f94af014789905aff8365f3ebf5 is 50, key is test_row_0/B:col10/1733217281381/Put/seqid=0 2024-12-03T09:14:43,157 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:43,161 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412032641a3b40b5445a2b977dbcbe737632e_7bd72df0dd602d0d382d15e837ff9272 store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:43,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742038_1214 (size=13119) 2024-12-03T09:14:43,171 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/d2e77f94af014789905aff8365f3ebf5 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d2e77f94af014789905aff8365f3ebf5 2024-12-03T09:14:43,178 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/B of 7bd72df0dd602d0d382d15e837ff9272 into d2e77f94af014789905aff8365f3ebf5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:43,178 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:43,178 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/B, priority=12, startTime=1733217283138; duration=0sec 2024-12-03T09:14:43,178 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:43,178 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:B 2024-12-03T09:14:43,178 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:14:43,180 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:14:43,180 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 7bd72df0dd602d0d382d15e837ff9272/C is initiating minor compaction (all files) 2024-12-03T09:14:43,180 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7bd72df0dd602d0d382d15e837ff9272/C in TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:43,181 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/8f5fe620ec484394b4d0f408b1c10ec9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/0137905c70074738a76d241a7cf9e3a7, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/475db233ba004bcaa0432c5592b382a7, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/6449690d3415498794e45c71cb892d36] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp, totalSize=48.7 K 2024-12-03T09:14:43,181 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f5fe620ec484394b4d0f408b1c10ec9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733217276332 2024-12-03T09:14:43,181 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0137905c70074738a76d241a7cf9e3a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1733217277533 2024-12-03T09:14:43,182 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 475db233ba004bcaa0432c5592b382a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1733217278524 2024-12-03T09:14:43,182 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6449690d3415498794e45c71cb892d36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733217279214 2024-12-03T09:14:43,191 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412032641a3b40b5445a2b977dbcbe737632e_7bd72df0dd602d0d382d15e837ff9272, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:43,191 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032641a3b40b5445a2b977dbcbe737632e_7bd72df0dd602d0d382d15e837ff9272 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:43,198 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#C#compaction#176 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:43,198 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/83e48025330d47b9b6517b2802b0afb2 is 50, key is test_row_0/C:col10/1733217281381/Put/seqid=0 2024-12-03T09:14:43,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742039_1215 (size=4469) 2024-12-03T09:14:43,201 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7bd72df0dd602d0d382d15e837ff9272#A#compaction#175 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:43,202 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3d8311e76b904098ade1bc4f8072a1f6 is 175, key is test_row_0/A:col10/1733217281381/Put/seqid=0 2024-12-03T09:14:43,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742040_1216 (size=13119) 2024-12-03T09:14:43,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742041_1217 (size=32073) 2024-12-03T09:14:43,209 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/83e48025330d47b9b6517b2802b0afb2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/83e48025330d47b9b6517b2802b0afb2 2024-12-03T09:14:43,214 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/C of 7bd72df0dd602d0d382d15e837ff9272 into 83e48025330d47b9b6517b2802b0afb2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:43,214 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:43,215 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/C, priority=12, startTime=1733217283138; duration=0sec 2024-12-03T09:14:43,215 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:43,215 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:C 2024-12-03T09:14:43,614 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/3d8311e76b904098ade1bc4f8072a1f6 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3d8311e76b904098ade1bc4f8072a1f6 2024-12-03T09:14:43,620 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7bd72df0dd602d0d382d15e837ff9272/A of 7bd72df0dd602d0d382d15e837ff9272 into 3d8311e76b904098ade1bc4f8072a1f6(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:43,620 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7bd72df0dd602d0d382d15e837ff9272: 2024-12-03T09:14:43,620 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., storeName=7bd72df0dd602d0d382d15e837ff9272/A, priority=12, startTime=1733217283137; duration=0sec 2024-12-03T09:14:43,620 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:43,620 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7bd72df0dd602d0d382d15e837ff9272:A 2024-12-03T09:14:48,254 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-12-03T09:14:48,254 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:48,254 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:48,254 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:48,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:48,331 INFO [Thread-607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-607. 2024-12-03T09:14:48,332 DEBUG [Thread-607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:48,332 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:48,332 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:48,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:48,352 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-12-03T09:14:48,352 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:14:48,352 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:48,352 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:48,352 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:48,352 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T09:14:48,352 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-12-03T09:14:48,352 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-12-03T09:14:48,352 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 166 2024-12-03T09:14:48,352 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-03T09:14:48,352 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-12-03T09:14:48,352 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T09:14:48,352 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4133 2024-12-03T09:14:48,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4091 2024-12-03T09:14:48,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T09:14:48,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1802 2024-12-03T09:14:48,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5406 rows 2024-12-03T09:14:48,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1800 2024-12-03T09:14:48,353 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5398 rows 2024-12-03T09:14:48,353 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T09:14:48,353 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:14:48,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:48,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:48,353 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:14:48,354 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T09:14:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T09:14:48,359 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217288358"}]},"ts":"1733217288358"} 2024-12-03T09:14:48,361 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T09:14:48,361 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T09:14:48,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T09:14:48,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, UNASSIGN}] 2024-12-03T09:14:48,365 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, UNASSIGN 2024-12-03T09:14:48,368 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=7bd72df0dd602d0d382d15e837ff9272, regionState=CLOSING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:48,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, UNASSIGN because future has completed 2024-12-03T09:14:48,370 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T09:14:48,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:14:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T09:14:48,526 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:48,526 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T09:14:48,526 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 7bd72df0dd602d0d382d15e837ff9272, disabling compactions & flushes 2024-12-03T09:14:48,526 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:48,527 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:48,527 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. after waiting 0 ms 2024-12-03T09:14:48,527 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:48,527 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 7bd72df0dd602d0d382d15e837ff9272 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-03T09:14:48,527 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=A 2024-12-03T09:14:48,527 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:48,527 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=B 2024-12-03T09:14:48,528 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:48,528 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7bd72df0dd602d0d382d15e837ff9272, store=C 2024-12-03T09:14:48,528 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:48,540 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032667b6b3b6b747c1b7402f41138440a2_7bd72df0dd602d0d382d15e837ff9272 is 50, key is test_row_0/A:col10/1733217288330/Put/seqid=0 2024-12-03T09:14:48,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742042_1218 (size=12454) 2024-12-03T09:14:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T09:14:48,945 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:48,956 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032667b6b3b6b747c1b7402f41138440a2_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032667b6b3b6b747c1b7402f41138440a2_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:48,957 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/42c7e4db762f47e9b694a89466a26bf9, store: [table=TestAcidGuarantees family=A region=7bd72df0dd602d0d382d15e837ff9272] 2024-12-03T09:14:48,958 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/42c7e4db762f47e9b694a89466a26bf9 is 175, key is test_row_0/A:col10/1733217288330/Put/seqid=0 2024-12-03T09:14:48,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742043_1219 (size=31255) 2024-12-03T09:14:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T09:14:49,364 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/42c7e4db762f47e9b694a89466a26bf9 2024-12-03T09:14:49,375 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/d5c066327b334b81afdeb3ac5acba033 is 50, key is test_row_0/B:col10/1733217288330/Put/seqid=0 2024-12-03T09:14:49,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742044_1220 (size=12301) 2024-12-03T09:14:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T09:14:49,780 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/d5c066327b334b81afdeb3ac5acba033 2024-12-03T09:14:49,795 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/f60201f0ae0c4bec9981fed4b00bf9d8 is 50, key is test_row_0/C:col10/1733217288330/Put/seqid=0 2024-12-03T09:14:49,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742045_1221 (size=12301) 2024-12-03T09:14:50,201 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/f60201f0ae0c4bec9981fed4b00bf9d8 2024-12-03T09:14:50,212 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/A/42c7e4db762f47e9b694a89466a26bf9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/42c7e4db762f47e9b694a89466a26bf9 2024-12-03T09:14:50,218 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/42c7e4db762f47e9b694a89466a26bf9, entries=150, sequenceid=393, filesize=30.5 K 2024-12-03T09:14:50,219 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/B/d5c066327b334b81afdeb3ac5acba033 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d5c066327b334b81afdeb3ac5acba033 2024-12-03T09:14:50,224 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d5c066327b334b81afdeb3ac5acba033, entries=150, sequenceid=393, filesize=12.0 K 2024-12-03T09:14:50,225 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/.tmp/C/f60201f0ae0c4bec9981fed4b00bf9d8 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/f60201f0ae0c4bec9981fed4b00bf9d8 2024-12-03T09:14:50,231 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/f60201f0ae0c4bec9981fed4b00bf9d8, entries=150, sequenceid=393, filesize=12.0 K 2024-12-03T09:14:50,232 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 7bd72df0dd602d0d382d15e837ff9272 in 1705ms, sequenceid=393, compaction requested=false 2024-12-03T09:14:50,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3fb94f46b0dc4c87b8ffc74711eebd7b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/b8713a54239f4619b0f1b826f5b76fed, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/12209cb8ec004ca1b285436b98a0cecf, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/185858289cf3430da4505d7eca519f86, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/fc9125b805fa4ab799fc7163ee750c90, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/35fd1c22cea24115a4e0bcab56537151, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/813937e3888449dbb88d0d68199737ac, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/383b03ffb7264985963586195a4da969, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/bba43dca7b4541678b1218cb72a4eb98, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/14b25c9ff1d2450eab9e076e4d15e006, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/54dc3deee5cc41aa808b1acd5a0a2ee2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/615426d677f0459583a52d43ce8ae97e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/8d74fa89f2f6407fb46a5abf9e4b3a68, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/5a9555de673c4f85aa2239b209480a57, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/84c18b12e232461cbf9f6be998cbb326, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/ab3a794ff5a442fca94cf6ac112b5b0d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/be67871c998443a1b29897849b07cd9d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/4b89e46d22734cff9bfd6df0a80ec120, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/e32afb18b32449d08f803c830c00a857, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/1dae3cf2ac56468cae58f1b1dd970678, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/2701d807f7dc45ddafc93fd38be59e2a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/f6430f4911d74555870c50b5eb357bca, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3c1ba9acba7d4e4093233678135b367d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3890c994b32148d1aa6cc749da4ec41e] to archive 2024-12-03T09:14:50,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:14:50,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3fb94f46b0dc4c87b8ffc74711eebd7b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3fb94f46b0dc4c87b8ffc74711eebd7b 2024-12-03T09:14:50,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/b8713a54239f4619b0f1b826f5b76fed to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/b8713a54239f4619b0f1b826f5b76fed 2024-12-03T09:14:50,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/12209cb8ec004ca1b285436b98a0cecf to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/12209cb8ec004ca1b285436b98a0cecf 2024-12-03T09:14:50,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/185858289cf3430da4505d7eca519f86 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/185858289cf3430da4505d7eca519f86 2024-12-03T09:14:50,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/fc9125b805fa4ab799fc7163ee750c90 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/fc9125b805fa4ab799fc7163ee750c90 2024-12-03T09:14:50,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/35fd1c22cea24115a4e0bcab56537151 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/35fd1c22cea24115a4e0bcab56537151 2024-12-03T09:14:50,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/813937e3888449dbb88d0d68199737ac to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/813937e3888449dbb88d0d68199737ac 2024-12-03T09:14:50,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/383b03ffb7264985963586195a4da969 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/383b03ffb7264985963586195a4da969 2024-12-03T09:14:50,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/bba43dca7b4541678b1218cb72a4eb98 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/bba43dca7b4541678b1218cb72a4eb98 2024-12-03T09:14:50,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/14b25c9ff1d2450eab9e076e4d15e006 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/14b25c9ff1d2450eab9e076e4d15e006 2024-12-03T09:14:50,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/54dc3deee5cc41aa808b1acd5a0a2ee2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/54dc3deee5cc41aa808b1acd5a0a2ee2 2024-12-03T09:14:50,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/615426d677f0459583a52d43ce8ae97e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/615426d677f0459583a52d43ce8ae97e 2024-12-03T09:14:50,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/8d74fa89f2f6407fb46a5abf9e4b3a68 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/8d74fa89f2f6407fb46a5abf9e4b3a68 2024-12-03T09:14:50,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/5a9555de673c4f85aa2239b209480a57 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/5a9555de673c4f85aa2239b209480a57 2024-12-03T09:14:50,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/84c18b12e232461cbf9f6be998cbb326 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/84c18b12e232461cbf9f6be998cbb326 2024-12-03T09:14:50,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/ab3a794ff5a442fca94cf6ac112b5b0d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/ab3a794ff5a442fca94cf6ac112b5b0d 2024-12-03T09:14:50,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/be67871c998443a1b29897849b07cd9d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/be67871c998443a1b29897849b07cd9d 2024-12-03T09:14:50,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/4b89e46d22734cff9bfd6df0a80ec120 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/4b89e46d22734cff9bfd6df0a80ec120 2024-12-03T09:14:50,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/e32afb18b32449d08f803c830c00a857 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/e32afb18b32449d08f803c830c00a857 2024-12-03T09:14:50,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/1dae3cf2ac56468cae58f1b1dd970678 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/1dae3cf2ac56468cae58f1b1dd970678 2024-12-03T09:14:50,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/2701d807f7dc45ddafc93fd38be59e2a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/2701d807f7dc45ddafc93fd38be59e2a 2024-12-03T09:14:50,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/f6430f4911d74555870c50b5eb357bca to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/f6430f4911d74555870c50b5eb357bca 2024-12-03T09:14:50,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3c1ba9acba7d4e4093233678135b367d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3c1ba9acba7d4e4093233678135b367d 2024-12-03T09:14:50,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3890c994b32148d1aa6cc749da4ec41e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3890c994b32148d1aa6cc749da4ec41e 2024-12-03T09:14:50,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/0c976b4a15f1454d93095fa4db0fa823, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/07d1ab28a5cc49fcb3ce335c3ae06c58, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/088efacc166a47f484d28645e36c779c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/3dbc6446009d47fab60625c7a03f88b9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/34698cf1ac0c48948fe68e964e5abde0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/387c9c444d7b4ed9a30ca9fa17737f61, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/57e80466db424f0e857025342b3cb53e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/1ba4b01c8efe425083674e47f8592f85, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/c23e2be9eea54d98924581dd29ccf06b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/9dc199f155584a2fa2d1670701831069, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/ec55df94ac644c509ef61b2e8d0f1ae9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/2dbd3f95b8e740d39aa35065b2ec9be6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d7cec446af5a40e78eaa586ade3c2dee, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/58b46adf60f54c56a82d301626def78d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/25b8ed91c22446c9a3bffde832dba615, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/27675729a6354481a8346692b1ccbd9c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/36ed7556fb4e402f9764d5e0a0c670bc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/62019d2f4a3348c7bdff81c58455459c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/6d93d3e930da486ba836b5dadb97e8a1, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/4013c47f55ff492e961cd8a3d01ad0a2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/eb2b9e67f75f461984b64048071f205a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/45cf5b31a5fb417a99969011797eb05a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/e91449bf95d340c08ea998d6c4e742aa, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d756696c174c4effaa71d08463b479b7] to archive 2024-12-03T09:14:50,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:14:50,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/0c976b4a15f1454d93095fa4db0fa823 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/0c976b4a15f1454d93095fa4db0fa823 2024-12-03T09:14:50,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/07d1ab28a5cc49fcb3ce335c3ae06c58 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/07d1ab28a5cc49fcb3ce335c3ae06c58 2024-12-03T09:14:50,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/088efacc166a47f484d28645e36c779c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/088efacc166a47f484d28645e36c779c 2024-12-03T09:14:50,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/3dbc6446009d47fab60625c7a03f88b9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/3dbc6446009d47fab60625c7a03f88b9 2024-12-03T09:14:50,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/34698cf1ac0c48948fe68e964e5abde0 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/34698cf1ac0c48948fe68e964e5abde0 2024-12-03T09:14:50,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/387c9c444d7b4ed9a30ca9fa17737f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/387c9c444d7b4ed9a30ca9fa17737f61 2024-12-03T09:14:50,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/57e80466db424f0e857025342b3cb53e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/57e80466db424f0e857025342b3cb53e 2024-12-03T09:14:50,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/1ba4b01c8efe425083674e47f8592f85 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/1ba4b01c8efe425083674e47f8592f85 2024-12-03T09:14:50,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/c23e2be9eea54d98924581dd29ccf06b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/c23e2be9eea54d98924581dd29ccf06b 2024-12-03T09:14:50,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/9dc199f155584a2fa2d1670701831069 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/9dc199f155584a2fa2d1670701831069 2024-12-03T09:14:50,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/ec55df94ac644c509ef61b2e8d0f1ae9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/ec55df94ac644c509ef61b2e8d0f1ae9 2024-12-03T09:14:50,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/2dbd3f95b8e740d39aa35065b2ec9be6 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/2dbd3f95b8e740d39aa35065b2ec9be6 2024-12-03T09:14:50,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d7cec446af5a40e78eaa586ade3c2dee to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d7cec446af5a40e78eaa586ade3c2dee 2024-12-03T09:14:50,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/58b46adf60f54c56a82d301626def78d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/58b46adf60f54c56a82d301626def78d 2024-12-03T09:14:50,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/25b8ed91c22446c9a3bffde832dba615 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/25b8ed91c22446c9a3bffde832dba615 2024-12-03T09:14:50,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/27675729a6354481a8346692b1ccbd9c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/27675729a6354481a8346692b1ccbd9c 2024-12-03T09:14:50,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/36ed7556fb4e402f9764d5e0a0c670bc to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/36ed7556fb4e402f9764d5e0a0c670bc 2024-12-03T09:14:50,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/62019d2f4a3348c7bdff81c58455459c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/62019d2f4a3348c7bdff81c58455459c 2024-12-03T09:14:50,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/6d93d3e930da486ba836b5dadb97e8a1 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/6d93d3e930da486ba836b5dadb97e8a1 2024-12-03T09:14:50,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/4013c47f55ff492e961cd8a3d01ad0a2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/4013c47f55ff492e961cd8a3d01ad0a2 2024-12-03T09:14:50,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/eb2b9e67f75f461984b64048071f205a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/eb2b9e67f75f461984b64048071f205a 2024-12-03T09:14:50,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/45cf5b31a5fb417a99969011797eb05a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/45cf5b31a5fb417a99969011797eb05a 2024-12-03T09:14:50,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/e91449bf95d340c08ea998d6c4e742aa to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/e91449bf95d340c08ea998d6c4e742aa 2024-12-03T09:14:50,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d756696c174c4effaa71d08463b479b7 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d756696c174c4effaa71d08463b479b7 2024-12-03T09:14:50,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/e2e333af13184ee993ae09ec0abdf3d0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/20c33723c8aa4ea0ab158254d6824b05, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/bc8b32bc223e4b7a80dc8bd03a67b703, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/ee5f5e330a124569aa666d7bb91e81fe, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/c9eb0cd6e3a14c019df625927700128e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/496ce4e3c49e493aaa2c554989a8dd9b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/1b52211c85b64496aaa3ea521a20456b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/b743d873cd8e4960872f7ba7662934da, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/f2971ee93ba94d48938d9041d43bc66f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/b81dc79b8dd14d6ea2dfc225a866bc58, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/bc61627cf98d46649e0f14e08a001d9f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/6a023500dad94237bb700301fe0503ff, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/5e522115f79f4e65877a208b63e2c2e4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/854346728a334d4e9b595d76868f7a30, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/2929794bd0de42069c321854e2a624ba, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/709a4f06cfe5477dbcda269725c48dcb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/30c1b33645d144da8d9b92f2a5c5b632, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/885f64175a3344c29460c7b8a0b9ce69, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/1f9875a6e45f4bf19925d8007cf2e13f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/8f5fe620ec484394b4d0f408b1c10ec9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/173440c98c454b8c81cd861471ab2209, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/0137905c70074738a76d241a7cf9e3a7, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/475db233ba004bcaa0432c5592b382a7, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/6449690d3415498794e45c71cb892d36] to archive 2024-12-03T09:14:50,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:14:50,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/e2e333af13184ee993ae09ec0abdf3d0 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/e2e333af13184ee993ae09ec0abdf3d0 2024-12-03T09:14:50,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/20c33723c8aa4ea0ab158254d6824b05 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/20c33723c8aa4ea0ab158254d6824b05 2024-12-03T09:14:50,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/bc8b32bc223e4b7a80dc8bd03a67b703 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/bc8b32bc223e4b7a80dc8bd03a67b703 2024-12-03T09:14:50,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/ee5f5e330a124569aa666d7bb91e81fe to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/ee5f5e330a124569aa666d7bb91e81fe 2024-12-03T09:14:50,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/c9eb0cd6e3a14c019df625927700128e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/c9eb0cd6e3a14c019df625927700128e 2024-12-03T09:14:50,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/496ce4e3c49e493aaa2c554989a8dd9b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/496ce4e3c49e493aaa2c554989a8dd9b 2024-12-03T09:14:50,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/1b52211c85b64496aaa3ea521a20456b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/1b52211c85b64496aaa3ea521a20456b 2024-12-03T09:14:50,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/b743d873cd8e4960872f7ba7662934da to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/b743d873cd8e4960872f7ba7662934da 2024-12-03T09:14:50,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/f2971ee93ba94d48938d9041d43bc66f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/f2971ee93ba94d48938d9041d43bc66f 2024-12-03T09:14:50,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/b81dc79b8dd14d6ea2dfc225a866bc58 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/b81dc79b8dd14d6ea2dfc225a866bc58 2024-12-03T09:14:50,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/bc61627cf98d46649e0f14e08a001d9f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/bc61627cf98d46649e0f14e08a001d9f 2024-12-03T09:14:50,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/6a023500dad94237bb700301fe0503ff to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/6a023500dad94237bb700301fe0503ff 2024-12-03T09:14:50,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/5e522115f79f4e65877a208b63e2c2e4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/5e522115f79f4e65877a208b63e2c2e4 2024-12-03T09:14:50,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/854346728a334d4e9b595d76868f7a30 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/854346728a334d4e9b595d76868f7a30 2024-12-03T09:14:50,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/2929794bd0de42069c321854e2a624ba to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/2929794bd0de42069c321854e2a624ba 2024-12-03T09:14:50,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/709a4f06cfe5477dbcda269725c48dcb to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/709a4f06cfe5477dbcda269725c48dcb 2024-12-03T09:14:50,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/30c1b33645d144da8d9b92f2a5c5b632 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/30c1b33645d144da8d9b92f2a5c5b632 2024-12-03T09:14:50,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/885f64175a3344c29460c7b8a0b9ce69 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/885f64175a3344c29460c7b8a0b9ce69 2024-12-03T09:14:50,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/1f9875a6e45f4bf19925d8007cf2e13f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/1f9875a6e45f4bf19925d8007cf2e13f 2024-12-03T09:14:50,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/8f5fe620ec484394b4d0f408b1c10ec9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/8f5fe620ec484394b4d0f408b1c10ec9 2024-12-03T09:14:50,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/173440c98c454b8c81cd861471ab2209 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/173440c98c454b8c81cd861471ab2209 2024-12-03T09:14:50,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/0137905c70074738a76d241a7cf9e3a7 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/0137905c70074738a76d241a7cf9e3a7 2024-12-03T09:14:50,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/475db233ba004bcaa0432c5592b382a7 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/475db233ba004bcaa0432c5592b382a7 2024-12-03T09:14:50,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/6449690d3415498794e45c71cb892d36 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/6449690d3415498794e45c71cb892d36 2024-12-03T09:14:50,324 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/recovered.edits/396.seqid, newMaxSeqId=396, maxSeqId=4 2024-12-03T09:14:50,325 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. 2024-12-03T09:14:50,325 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 7bd72df0dd602d0d382d15e837ff9272: Waiting for close lock at 1733217288526Running coprocessor pre-close hooks at 1733217288526Disabling compacts and flushes for region at 1733217288526Disabling writes for close at 1733217288527 (+1 ms)Obtaining lock to block concurrent updates at 1733217288527Preparing flush snapshotting stores in 7bd72df0dd602d0d382d15e837ff9272 at 1733217288527Finished memstore snapshotting TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733217288528 (+1 ms)Flushing stores of TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272. at 1733217288529 (+1 ms)Flushing 7bd72df0dd602d0d382d15e837ff9272/A: creating writer at 1733217288529Flushing mob file 7bd72df0dd602d0d382d15e837ff9272/A: appending metadata at 1733217288539 (+10 ms)Flushing mob file 7bd72df0dd602d0d382d15e837ff9272/A: closing flushed file at 1733217288540 (+1 ms)Flushing 7bd72df0dd602d0d382d15e837ff9272/A: appending metadata at 1733217288957 (+417 ms)Flushing 7bd72df0dd602d0d382d15e837ff9272/A: closing flushed file at 1733217288957Flushing 7bd72df0dd602d0d382d15e837ff9272/B: creating writer at 1733217289370 (+413 ms)Flushing 7bd72df0dd602d0d382d15e837ff9272/B: appending metadata at 1733217289374 (+4 ms)Flushing 7bd72df0dd602d0d382d15e837ff9272/B: closing flushed file at 1733217289374Flushing 7bd72df0dd602d0d382d15e837ff9272/C: creating writer at 1733217289791 (+417 ms)Flushing 7bd72df0dd602d0d382d15e837ff9272/C: appending metadata at 1733217289794 (+3 ms)Flushing 7bd72df0dd602d0d382d15e837ff9272/C: closing flushed file at 1733217289794Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@378a41cd: reopening flushed file at 1733217290211 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18cc23dc: reopening flushed file at 1733217290218 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42d68203: reopening flushed file at 1733217290224 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 7bd72df0dd602d0d382d15e837ff9272 in 1705ms, sequenceid=393, compaction requested=false at 1733217290232 (+8 ms)Writing region close event to WAL at 1733217290321 (+89 ms)Running coprocessor post-close hooks at 1733217290325 (+4 ms)Closed at 1733217290325 2024-12-03T09:14:50,327 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,327 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=7bd72df0dd602d0d382d15e837ff9272, regionState=CLOSED 2024-12-03T09:14:50,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:14:50,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-03T09:14:50,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 7bd72df0dd602d0d382d15e837ff9272, server=699c9baf3d29,42271,1733217224446 in 1.9590 sec 2024-12-03T09:14:50,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-03T09:14:50,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7bd72df0dd602d0d382d15e837ff9272, UNASSIGN in 1.9660 sec 2024-12-03T09:14:50,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-03T09:14:50,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9710 sec 2024-12-03T09:14:50,334 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217290334"}]},"ts":"1733217290334"} 2024-12-03T09:14:50,336 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T09:14:50,336 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T09:14:50,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9820 sec 2024-12-03T09:14:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T09:14:50,510 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:50,511 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T09:14:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:50,515 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T09:14:50,517 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:50,521 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,525 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/recovered.edits] 2024-12-03T09:14:50,529 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3d8311e76b904098ade1bc4f8072a1f6 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/3d8311e76b904098ade1bc4f8072a1f6 2024-12-03T09:14:50,531 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/42c7e4db762f47e9b694a89466a26bf9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/A/42c7e4db762f47e9b694a89466a26bf9 2024-12-03T09:14:50,534 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d2e77f94af014789905aff8365f3ebf5 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d2e77f94af014789905aff8365f3ebf5 2024-12-03T09:14:50,535 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d5c066327b334b81afdeb3ac5acba033 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/B/d5c066327b334b81afdeb3ac5acba033 2024-12-03T09:14:50,537 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/83e48025330d47b9b6517b2802b0afb2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/83e48025330d47b9b6517b2802b0afb2 2024-12-03T09:14:50,538 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/f60201f0ae0c4bec9981fed4b00bf9d8 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/C/f60201f0ae0c4bec9981fed4b00bf9d8 2024-12-03T09:14:50,541 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/recovered.edits/396.seqid to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272/recovered.edits/396.seqid 2024-12-03T09:14:50,541 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,541 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T09:14:50,542 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T09:14:50,542 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T09:14:50,546 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031be4b09d948d477b9fb9c3562dea3e8f_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031be4b09d948d477b9fb9c3562dea3e8f_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,547 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031e2dbb88a00b46c681a5c04ba031a25b_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031e2dbb88a00b46c681a5c04ba031a25b_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,548 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032667b6b3b6b747c1b7402f41138440a2_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032667b6b3b6b747c1b7402f41138440a2_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,549 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120329a302ad2c2a4eea86e28da4dd49a1ab_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120329a302ad2c2a4eea86e28da4dd49a1ab_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,550 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033912a14ecaf2405db602de6142b49eab_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033912a14ecaf2405db602de6142b49eab_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,551 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034763c4834c7e45158640e087f2d50f3c_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034763c4834c7e45158640e087f2d50f3c_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,552 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203524fc152bdfc4401acdb8abef6980354_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203524fc152bdfc4401acdb8abef6980354_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,553 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035f0d7b2ec07045c1afd1dba50f67b378_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035f0d7b2ec07045c1afd1dba50f67b378_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,553 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203682c8d8b2f2b47c3b4072b09f80127e1_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203682c8d8b2f2b47c3b4072b09f80127e1_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,554 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036fa084d765604a89bb90316be24baf57_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036fa084d765604a89bb90316be24baf57_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,555 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203833c3be873d043e1b3e5f188d157df03_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203833c3be873d043e1b3e5f188d157df03_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,556 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039a905eac3897417293bff6cc34c5d15d_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039a905eac3897417293bff6cc34c5d15d_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,557 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b6717a1b62a84e739993641a08961888_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b6717a1b62a84e739993641a08961888_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,558 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c5c750be3b4340f5a2b7ef668daf5c2f_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203c5c750be3b4340f5a2b7ef668daf5c2f_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,559 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ced10ce0f0534a83b35be4e3e979bb2f_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ced10ce0f0534a83b35be4e3e979bb2f_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,560 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d8c15a5d5ef44c2a93254f814dd0ca48_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d8c15a5d5ef44c2a93254f814dd0ca48_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,561 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203decd3811801047aea138e64bfa2910f4_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203decd3811801047aea138e64bfa2910f4_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,562 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e2d8ccb51d7b4772b08b78c4eafafb1e_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e2d8ccb51d7b4772b08b78c4eafafb1e_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,563 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f8e10385c66e4b71adeae3147cd1f10f_7bd72df0dd602d0d382d15e837ff9272 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f8e10385c66e4b71adeae3147cd1f10f_7bd72df0dd602d0d382d15e837ff9272 2024-12-03T09:14:50,564 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T09:14:50,566 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:50,568 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T09:14:50,570 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T09:14:50,571 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:50,572 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T09:14:50,572 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733217290572"}]},"ts":"9223372036854775807"} 2024-12-03T09:14:50,575 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T09:14:50,575 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7bd72df0dd602d0d382d15e837ff9272, NAME => 'TestAcidGuarantees,,1733217256575.7bd72df0dd602d0d382d15e837ff9272.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T09:14:50,575 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T09:14:50,575 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733217290575"}]},"ts":"9223372036854775807"} 2024-12-03T09:14:50,577 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T09:14:50,578 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:50,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-12-03T09:14:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T09:14:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T09:14:50,620 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:50,629 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 249), OpenFileDescriptor=453 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=363 (was 341) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12950 (was 13136) 2024-12-03T09:14:50,637 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=363, ProcessCount=11, AvailableMemoryMB=12946 2024-12-03T09:14:50,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T09:14:50,639 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T09:14:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T09:14:50,641 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T09:14:50,641 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:50,642 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-12-03T09:14:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T09:14:50,642 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T09:14:50,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742046_1222 (size=1061) 2024-12-03T09:14:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T09:14:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T09:14:51,050 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09 2024-12-03T09:14:51,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742047_1223 (size=53) 2024-12-03T09:14:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T09:14:51,458 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:14:51,458 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 58a142609061c77936f842676cc6c10e, disabling compactions & flushes 2024-12-03T09:14:51,458 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:51,459 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:51,459 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. after waiting 0 ms 2024-12-03T09:14:51,459 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:51,459 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:51,459 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 58a142609061c77936f842676cc6c10e: Waiting for close lock at 1733217291458Disabling compacts and flushes for region at 1733217291458Disabling writes for close at 1733217291459 (+1 ms)Writing region close event to WAL at 1733217291459Closed at 1733217291459 2024-12-03T09:14:51,462 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T09:14:51,462 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733217291462"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733217291462"}]},"ts":"1733217291462"} 2024-12-03T09:14:51,465 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T09:14:51,467 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T09:14:51,467 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217291467"}]},"ts":"1733217291467"} 2024-12-03T09:14:51,469 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T09:14:51,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58a142609061c77936f842676cc6c10e, ASSIGN}] 2024-12-03T09:14:51,470 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58a142609061c77936f842676cc6c10e, ASSIGN 2024-12-03T09:14:51,470 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58a142609061c77936f842676cc6c10e, ASSIGN; state=OFFLINE, location=699c9baf3d29,42271,1733217224446; forceNewPlan=false, retain=false 2024-12-03T09:14:51,622 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=58a142609061c77936f842676cc6c10e, regionState=OPENING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:51,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58a142609061c77936f842676cc6c10e, ASSIGN because future has completed 2024-12-03T09:14:51,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:14:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T09:14:51,788 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:51,789 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:14:51,789 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:51,790 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:14:51,790 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:51,790 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:51,792 INFO [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:51,794 INFO [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:14:51,795 INFO [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58a142609061c77936f842676cc6c10e columnFamilyName A 2024-12-03T09:14:51,795 DEBUG [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:51,796 INFO [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] regionserver.HStore(327): Store=58a142609061c77936f842676cc6c10e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:14:51,796 INFO [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:51,798 INFO [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:14:51,798 INFO [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58a142609061c77936f842676cc6c10e columnFamilyName B 2024-12-03T09:14:51,798 DEBUG [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:51,799 INFO [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] regionserver.HStore(327): Store=58a142609061c77936f842676cc6c10e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:14:51,799 INFO [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:51,800 INFO [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:14:51,801 INFO [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58a142609061c77936f842676cc6c10e columnFamilyName C 2024-12-03T09:14:51,801 DEBUG [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:14:51,801 INFO [StoreOpener-58a142609061c77936f842676cc6c10e-1 {}] regionserver.HStore(327): Store=58a142609061c77936f842676cc6c10e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:14:51,801 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:51,801 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:51,802 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e 2024-12-03T09:14:51,803 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e 2024-12-03T09:14:51,804 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:51,804 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:51,805 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T09:14:51,807 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:51,810 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:14:51,810 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 58a142609061c77936f842676cc6c10e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66216274, jitterRate=-0.01330062747001648}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T09:14:51,810 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:51,811 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 58a142609061c77936f842676cc6c10e: Running coprocessor pre-open hook at 1733217291790Writing region info on filesystem at 1733217291790Initializing all the Stores at 1733217291792 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217291792Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217291792Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217291792Cleaning up temporary data from old regions at 1733217291804 (+12 ms)Running coprocessor post-open hooks at 1733217291810 (+6 ms)Region opened successfully at 1733217291811 (+1 ms) 2024-12-03T09:14:51,812 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., pid=65, masterSystemTime=1733217291781 2024-12-03T09:14:51,815 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:51,815 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:51,815 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=58a142609061c77936f842676cc6c10e, regionState=OPEN, openSeqNum=2, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:51,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:14:51,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-03T09:14:51,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 in 192 msec 2024-12-03T09:14:51,822 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-03T09:14:51,822 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58a142609061c77936f842676cc6c10e, ASSIGN in 352 msec 2024-12-03T09:14:51,823 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T09:14:51,823 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217291823"}]},"ts":"1733217291823"} 2024-12-03T09:14:51,824 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T09:14:51,825 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T09:14:51,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-12-03T09:14:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T09:14:52,800 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:52,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75850ca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:52,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:52,802 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587125f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:52,805 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d72d05c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,806 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:52,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:52,810 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T09:14:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 699c9baf3d29,35781,1733217223715 2024-12-03T09:14:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c187f19 2024-12-03T09:14:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T09:14:52,813 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T09:14:52,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c01eb36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:52,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:52,816 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339959da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:52,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:52,818 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d911d99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41eec903, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:52,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:52,821 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81dacfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:52,824 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:52,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749244c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@397324ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:52,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:52,826 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ee1b739, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:52,828 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@454e17d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77bd2bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:52,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:52,830 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276b3456, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:52,832 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a9e6186, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4689d9b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:52,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:52,834 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f1c5231, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:52,835 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ee3a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12026b1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:52,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:52,837 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:52,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:52,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:52,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d234b41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:52,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:52,838 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6755e399, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d1a27c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,839 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:52,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:52,840 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@136e0f77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:52,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:52,842 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:52,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13acb63e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7928d378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,843 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:52,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:52,843 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260bcf6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:52,845 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:52,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63455081, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4512988d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:52,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:52,846 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23de9b35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:52,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:52,848 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:52,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@164ff8dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656b74aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:14:52,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:14:52,849 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:14:52,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:14:52,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:14:52,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3b8ba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:14:52,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:14:52,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:14:52,850 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:14:52,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127a04ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:14:52,851 DEBUG [Thread-1064 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:52,851 DEBUG [Thread-1070 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:52,851 DEBUG [Thread-1059 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:52,852 DEBUG [Thread-1055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:52,852 DEBUG [Thread-1062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:52,852 DEBUG [Thread-1066 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:52,852 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:52,852 DEBUG [Thread-1057 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:52,853 DEBUG [Thread-1051 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-03T09:14:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:52,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T09:14:52,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:52,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:52,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:52,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:52,855 DEBUG [Thread-1068 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:52,856 DEBUG [Thread-1053 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:52,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:52,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:52,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:14:52,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:14:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:14:52,857 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:14:52,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:14:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:14:52,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:14:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:14:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:14:52,860 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:52,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:14:52,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:14:52,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:52,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:14:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:14:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:52,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:14:52,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:14:52,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:52,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:14:52,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:52,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:14:52,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:52,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/017b709515334f99956447b9b40b8c8a is 50, key is test_row_0/A:col10/1733217292862/Put/seqid=0 2024-12-03T09:14:52,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742048_1224 (size=12001) 2024-12-03T09:14:52,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217292895, max timestamp=1733217292905Num uniques 859; 2024-12-03T09:14:52,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217292895, max timestamp=1733217292904Num uniques 859; 2024-12-03T09:14:52,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:52,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:52,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217292895, max timestamp=1733217292905Num uniques 859; 2024-12-03T09:14:52,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:14:52,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217352952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:52,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:52,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:52,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:52,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217352953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:52,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:52,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:52,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217352953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:52,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:52,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:52,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:52,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T09:14:52,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217352955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:52,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217352955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T09:14:53,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:53,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:14:53,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:53,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217353059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217353069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217353070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217353070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217353070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T09:14:53,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:53,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:14:53,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:53,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T09:14:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217353270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217353279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217353279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217353279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217353279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/017b709515334f99956447b9b40b8c8a 2024-12-03T09:14:53,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T09:14:53,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:53,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:14:53,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:53,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/1058e7db7d214b8abc368427a3f2b3cd is 50, key is test_row_0/B:col10/1733217292862/Put/seqid=0 2024-12-03T09:14:53,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742049_1225 (size=12001) 2024-12-03T09:14:53,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T09:14:53,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T09:14:53,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:53,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:14:53,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:53,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217353579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217353589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217353589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217353589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217353589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:53,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:53,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T09:14:53,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:53,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:14:53,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:53,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/1058e7db7d214b8abc368427a3f2b3cd 2024-12-03T09:14:53,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/447ef3e72dd443199ad31d8eca958524 is 50, key is test_row_0/C:col10/1733217292862/Put/seqid=0 2024-12-03T09:14:53,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T09:14:53,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:53,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:14:53,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:53,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:14:53,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742050_1226 (size=12001) 2024-12-03T09:14:53,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/447ef3e72dd443199ad31d8eca958524 2024-12-03T09:14:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/017b709515334f99956447b9b40b8c8a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/017b709515334f99956447b9b40b8c8a 2024-12-03T09:14:53,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/017b709515334f99956447b9b40b8c8a, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T09:14:53,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/1058e7db7d214b8abc368427a3f2b3cd as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/1058e7db7d214b8abc368427a3f2b3cd 2024-12-03T09:14:53,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/1058e7db7d214b8abc368427a3f2b3cd, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T09:14:53,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/447ef3e72dd443199ad31d8eca958524 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/447ef3e72dd443199ad31d8eca958524 2024-12-03T09:14:53,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/447ef3e72dd443199ad31d8eca958524, entries=150, sequenceid=12, filesize=11.7 K 2024-12-03T09:14:53,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 58a142609061c77936f842676cc6c10e in 1002ms, sequenceid=12, compaction requested=false 2024-12-03T09:14:53,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:14:53,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T09:14:53,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:53,975 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T09:14:53,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:14:53,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:53,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:14:53,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:53,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:14:53,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:53,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/5cb899849ff24b8b8fa4f449f7f488b6 is 50, key is test_row_0/A:col10/1733217292948/Put/seqid=0 2024-12-03T09:14:53,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742051_1227 (size=12001) 2024-12-03T09:14:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T09:14:54,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:14:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:54,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217354103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217354103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217354104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217354104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217354105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217354210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217354210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217354210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217354219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217354219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,385 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/5cb899849ff24b8b8fa4f449f7f488b6 2024-12-03T09:14:54,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/7c59d0197a1d499d99a7c882ad260251 is 50, key is test_row_0/B:col10/1733217292948/Put/seqid=0 2024-12-03T09:14:54,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742052_1228 (size=12001) 2024-12-03T09:14:54,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217354419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217354419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217354419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217354429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217354429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217354730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217354730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217354730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217354739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217354739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:54,802 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/7c59d0197a1d499d99a7c882ad260251 2024-12-03T09:14:54,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/8cb1e4ad2ce24ffeb61c85ef16bc6350 is 50, key is test_row_0/C:col10/1733217292948/Put/seqid=0 2024-12-03T09:14:54,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742053_1229 (size=12001) 2024-12-03T09:14:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T09:14:55,217 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/8cb1e4ad2ce24ffeb61c85ef16bc6350 2024-12-03T09:14:55,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/5cb899849ff24b8b8fa4f449f7f488b6 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5cb899849ff24b8b8fa4f449f7f488b6 2024-12-03T09:14:55,226 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5cb899849ff24b8b8fa4f449f7f488b6, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T09:14:55,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/7c59d0197a1d499d99a7c882ad260251 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/7c59d0197a1d499d99a7c882ad260251 2024-12-03T09:14:55,231 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/7c59d0197a1d499d99a7c882ad260251, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T09:14:55,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/8cb1e4ad2ce24ffeb61c85ef16bc6350 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/8cb1e4ad2ce24ffeb61c85ef16bc6350 2024-12-03T09:14:55,236 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/8cb1e4ad2ce24ffeb61c85ef16bc6350, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T09:14:55,237 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 58a142609061c77936f842676cc6c10e in 1262ms, sequenceid=48, compaction requested=false 2024-12-03T09:14:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:14:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-03T09:14:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-03T09:14:55,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-03T09:14:55,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3780 sec 2024-12-03T09:14:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:55,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T09:14:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:14:55,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.3890 sec 2024-12-03T09:14:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:14:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:14:55,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:55,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/c95c3fe4a6c14a9ba4f6fee1524fa5ca is 50, key is test_row_0/A:col10/1733217295241/Put/seqid=0 2024-12-03T09:14:55,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742054_1230 (size=12001) 2024-12-03T09:14:55,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217355256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217355258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217355259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217355260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217355260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217355370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217355370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217355370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217355370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217355370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217355580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217355580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217355580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217355580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217355580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/c95c3fe4a6c14a9ba4f6fee1524fa5ca 2024-12-03T09:14:55,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/d15719a385f74e1a822fa998192dcb1b is 50, key is test_row_0/B:col10/1733217295241/Put/seqid=0 2024-12-03T09:14:55,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742055_1231 (size=12001) 2024-12-03T09:14:55,845 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T09:14:55,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217355890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217355899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217355900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217355900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217355900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:56,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/d15719a385f74e1a822fa998192dcb1b 2024-12-03T09:14:56,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/b00c9bfacbaa4144aa3678d4511c72d7 is 50, key is test_row_0/C:col10/1733217295241/Put/seqid=0 2024-12-03T09:14:56,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742056_1232 (size=12001) 2024-12-03T09:14:56,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217356399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:56,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217356409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:56,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217356409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:56,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217356410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:56,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217356410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:56,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/b00c9bfacbaa4144aa3678d4511c72d7 2024-12-03T09:14:56,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/c95c3fe4a6c14a9ba4f6fee1524fa5ca as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/c95c3fe4a6c14a9ba4f6fee1524fa5ca 2024-12-03T09:14:56,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/c95c3fe4a6c14a9ba4f6fee1524fa5ca, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T09:14:56,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/d15719a385f74e1a822fa998192dcb1b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/d15719a385f74e1a822fa998192dcb1b 2024-12-03T09:14:56,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/d15719a385f74e1a822fa998192dcb1b, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T09:14:56,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/b00c9bfacbaa4144aa3678d4511c72d7 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/b00c9bfacbaa4144aa3678d4511c72d7 2024-12-03T09:14:56,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/b00c9bfacbaa4144aa3678d4511c72d7, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T09:14:56,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 58a142609061c77936f842676cc6c10e in 1268ms, sequenceid=62, compaction requested=true 2024-12-03T09:14:56,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:14:56,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:14:56,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:56,510 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:56,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:14:56,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:56,510 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:56,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:14:56,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:56,511 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:56,511 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:56,511 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/B is initiating minor compaction (all files) 2024-12-03T09:14:56,511 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/A is initiating minor compaction (all files) 2024-12-03T09:14:56,511 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/B in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:56,511 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/A in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:56,511 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/017b709515334f99956447b9b40b8c8a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5cb899849ff24b8b8fa4f449f7f488b6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/c95c3fe4a6c14a9ba4f6fee1524fa5ca] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=35.2 K 2024-12-03T09:14:56,511 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/1058e7db7d214b8abc368427a3f2b3cd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/7c59d0197a1d499d99a7c882ad260251, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/d15719a385f74e1a822fa998192dcb1b] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=35.2 K 2024-12-03T09:14:56,512 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1058e7db7d214b8abc368427a3f2b3cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733217292862 2024-12-03T09:14:56,512 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 017b709515334f99956447b9b40b8c8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733217292862 2024-12-03T09:14:56,512 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb899849ff24b8b8fa4f449f7f488b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733217292948 2024-12-03T09:14:56,512 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c59d0197a1d499d99a7c882ad260251, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733217292948 2024-12-03T09:14:56,512 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c95c3fe4a6c14a9ba4f6fee1524fa5ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733217294103 2024-12-03T09:14:56,513 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting d15719a385f74e1a822fa998192dcb1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733217294103 2024-12-03T09:14:56,521 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#B#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:56,522 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/1c54e983214149f9914fe4412ee5a2fb is 50, key is test_row_0/B:col10/1733217295241/Put/seqid=0 2024-12-03T09:14:56,522 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#A#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:56,523 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/aac2417c16114c5eaf8b60af845529db is 50, key is test_row_0/A:col10/1733217295241/Put/seqid=0 2024-12-03T09:14:56,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742057_1233 (size=12104) 2024-12-03T09:14:56,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742058_1234 (size=12104) 2024-12-03T09:14:56,547 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/aac2417c16114c5eaf8b60af845529db as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/aac2417c16114c5eaf8b60af845529db 2024-12-03T09:14:56,556 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/A of 58a142609061c77936f842676cc6c10e into aac2417c16114c5eaf8b60af845529db(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:56,557 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:14:56,557 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/A, priority=13, startTime=1733217296509; duration=0sec 2024-12-03T09:14:56,557 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:14:56,557 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:A 2024-12-03T09:14:56,557 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:14:56,558 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:14:56,558 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/C is initiating minor compaction (all files) 2024-12-03T09:14:56,558 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/C in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:56,559 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/447ef3e72dd443199ad31d8eca958524, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/8cb1e4ad2ce24ffeb61c85ef16bc6350, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/b00c9bfacbaa4144aa3678d4511c72d7] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=35.2 K 2024-12-03T09:14:56,560 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 447ef3e72dd443199ad31d8eca958524, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733217292862 2024-12-03T09:14:56,560 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cb1e4ad2ce24ffeb61c85ef16bc6350, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733217292948 2024-12-03T09:14:56,560 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b00c9bfacbaa4144aa3678d4511c72d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733217294103 2024-12-03T09:14:56,575 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#C#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:14:56,575 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/21b0da63b13a473da7e2b16436e0b217 is 50, key is test_row_0/C:col10/1733217295241/Put/seqid=0 2024-12-03T09:14:56,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742059_1235 (size=12104) 2024-12-03T09:14:56,627 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/21b0da63b13a473da7e2b16436e0b217 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/21b0da63b13a473da7e2b16436e0b217 2024-12-03T09:14:56,635 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/C of 58a142609061c77936f842676cc6c10e into 21b0da63b13a473da7e2b16436e0b217(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:56,635 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:14:56,635 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/C, priority=13, startTime=1733217296510; duration=0sec 2024-12-03T09:14:56,635 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:56,635 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:C 2024-12-03T09:14:56,941 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/1c54e983214149f9914fe4412ee5a2fb as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/1c54e983214149f9914fe4412ee5a2fb 2024-12-03T09:14:56,947 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/B of 58a142609061c77936f842676cc6c10e into 1c54e983214149f9914fe4412ee5a2fb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:14:56,947 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:14:56,947 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/B, priority=13, startTime=1733217296510; duration=0sec 2024-12-03T09:14:56,947 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:14:56,947 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:B 2024-12-03T09:14:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T09:14:57,020 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:57,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:57,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-03T09:14:57,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T09:14:57,023 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:57,024 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:57,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T09:14:57,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T09:14:57,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:57,177 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T09:14:57,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:14:57,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:57,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:14:57,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:57,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:14:57,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:57,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/4ff1c63649b7446bb5e732cf478a26bd is 50, key is test_row_0/A:col10/1733217295252/Put/seqid=0 2024-12-03T09:14:57,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742060_1236 (size=12001) 2024-12-03T09:14:57,218 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/4ff1c63649b7446bb5e732cf478a26bd 2024-12-03T09:14:57,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/ca1de98dc89c4cf9952963469f563893 is 50, key is test_row_0/B:col10/1733217295252/Put/seqid=0 2024-12-03T09:14:57,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742061_1237 (size=12001) 2024-12-03T09:14:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T09:14:57,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:14:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:57,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:57,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217357428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217357428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:57,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217357429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:57,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217357430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:57,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217357430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:57,433 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:57,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:57,434 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:57,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217357540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:57,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217357540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:57,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217357540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:57,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T09:14:57,662 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/ca1de98dc89c4cf9952963469f563893 2024-12-03T09:14:57,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/c3a4e6f09f5446c48ba46adac9808757 is 50, key is test_row_0/C:col10/1733217295252/Put/seqid=0 2024-12-03T09:14:57,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742062_1238 (size=12001) 2024-12-03T09:14:57,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217357750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:57,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217357750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217357750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:58,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217358059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:58,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217358059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:58,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217358059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:58,074 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/c3a4e6f09f5446c48ba46adac9808757 2024-12-03T09:14:58,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/4ff1c63649b7446bb5e732cf478a26bd as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/4ff1c63649b7446bb5e732cf478a26bd 2024-12-03T09:14:58,083 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/4ff1c63649b7446bb5e732cf478a26bd, entries=150, sequenceid=84, filesize=11.7 K 2024-12-03T09:14:58,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/ca1de98dc89c4cf9952963469f563893 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/ca1de98dc89c4cf9952963469f563893 2024-12-03T09:14:58,087 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/ca1de98dc89c4cf9952963469f563893, entries=150, sequenceid=84, filesize=11.7 K 2024-12-03T09:14:58,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/c3a4e6f09f5446c48ba46adac9808757 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/c3a4e6f09f5446c48ba46adac9808757 2024-12-03T09:14:58,093 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/c3a4e6f09f5446c48ba46adac9808757, entries=150, sequenceid=84, filesize=11.7 K 2024-12-03T09:14:58,093 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 58a142609061c77936f842676cc6c10e in 916ms, sequenceid=84, compaction requested=false 2024-12-03T09:14:58,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:14:58,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:58,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-03T09:14:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-03T09:14:58,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-03T09:14:58,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0710 sec 2024-12-03T09:14:58,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.0770 sec 2024-12-03T09:14:58,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T09:14:58,160 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:14:58,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:14:58,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-03T09:14:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T09:14:58,163 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:14:58,164 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:14:58,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:14:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T09:14:58,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T09:14:58,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:58,317 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T09:14:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:14:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:14:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:14:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:14:58,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/94cc39cdba104dc5800e23ca8b8e1e98 is 50, key is test_row_0/A:col10/1733217297427/Put/seqid=0 2024-12-03T09:14:58,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742063_1239 (size=12001) 2024-12-03T09:14:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T09:14:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:58,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:14:58,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217358586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:58,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217358588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:58,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217358588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:58,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:58,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217358699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217358699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:58,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217358699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T09:14:58,794 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/94cc39cdba104dc5800e23ca8b8e1e98 2024-12-03T09:14:58,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/8bd05688007d458198644e3c3581130e is 50, key is test_row_0/B:col10/1733217297427/Put/seqid=0 2024-12-03T09:14:58,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742064_1240 (size=12001) 2024-12-03T09:14:58,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:58,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217358909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217358909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217358909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:58,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:59,207 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/8bd05688007d458198644e3c3581130e 2024-12-03T09:14:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/a996d1c899eb4eea93cc0d7a332cb08d is 50, key is test_row_0/C:col10/1733217297427/Put/seqid=0 2024-12-03T09:14:59,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217359219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:59,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217359219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217359219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:59,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742065_1241 (size=12001) 2024-12-03T09:14:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T09:14:59,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217359449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:59,451 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:59,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:59,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217359459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:59,462 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:14:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:59,627 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/a996d1c899eb4eea93cc0d7a332cb08d 2024-12-03T09:14:59,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/94cc39cdba104dc5800e23ca8b8e1e98 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/94cc39cdba104dc5800e23ca8b8e1e98 2024-12-03T09:14:59,635 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/94cc39cdba104dc5800e23ca8b8e1e98, entries=150, sequenceid=97, filesize=11.7 K 2024-12-03T09:14:59,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/8bd05688007d458198644e3c3581130e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/8bd05688007d458198644e3c3581130e 2024-12-03T09:14:59,640 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/8bd05688007d458198644e3c3581130e, entries=150, sequenceid=97, filesize=11.7 K 2024-12-03T09:14:59,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/a996d1c899eb4eea93cc0d7a332cb08d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a996d1c899eb4eea93cc0d7a332cb08d 2024-12-03T09:14:59,646 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a996d1c899eb4eea93cc0d7a332cb08d, entries=150, sequenceid=97, filesize=11.7 K 2024-12-03T09:14:59,646 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 58a142609061c77936f842676cc6c10e in 1329ms, sequenceid=97, compaction requested=true 2024-12-03T09:14:59,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:14:59,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:14:59,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-03T09:14:59,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-03T09:14:59,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-03T09:14:59,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4840 sec 2024-12-03T09:14:59,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4900 sec 2024-12-03T09:14:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:14:59,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T09:14:59,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:14:59,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:59,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:14:59,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:59,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:14:59,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:14:59,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/4b1f412421784ab59dfed23f9cad19ce is 50, key is test_row_0/A:col10/1733217299731/Put/seqid=0 2024-12-03T09:14:59,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742066_1242 (size=16681) 2024-12-03T09:14:59,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217359742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:59,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217359743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:59,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217359746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:59,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217359849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:59,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217359849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:14:59,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:14:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217359860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:14:59,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:14:59,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:00,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217360059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217360059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:00,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:00,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217360069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:00,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/4b1f412421784ab59dfed23f9cad19ce 2024-12-03T09:15:00,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/9c762d669b7b4134bcf04b09a858ed2f is 50, key is test_row_0/B:col10/1733217299731/Put/seqid=0 2024-12-03T09:15:00,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742067_1243 (size=12001) 2024-12-03T09:15:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T09:15:00,309 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:00,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-03T09:15:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T09:15:00,311 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:00,312 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:00,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:00,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217360369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:00,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:00,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217360369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:00,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217360379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:00,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T09:15:00,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T09:15:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:00,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:00,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:00,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/9c762d669b7b4134bcf04b09a858ed2f 2024-12-03T09:15:00,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/2dab1da0a8024f41bca04ab69ac9b6ca is 50, key is test_row_0/C:col10/1733217299731/Put/seqid=0 2024-12-03T09:15:00,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742068_1244 (size=12001) 2024-12-03T09:15:00,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/2dab1da0a8024f41bca04ab69ac9b6ca 2024-12-03T09:15:00,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/4b1f412421784ab59dfed23f9cad19ce as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/4b1f412421784ab59dfed23f9cad19ce 2024-12-03T09:15:00,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/4b1f412421784ab59dfed23f9cad19ce, entries=250, sequenceid=119, filesize=16.3 K 2024-12-03T09:15:00,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/9c762d669b7b4134bcf04b09a858ed2f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/9c762d669b7b4134bcf04b09a858ed2f 2024-12-03T09:15:00,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/9c762d669b7b4134bcf04b09a858ed2f, entries=150, sequenceid=119, filesize=11.7 K 2024-12-03T09:15:00,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/2dab1da0a8024f41bca04ab69ac9b6ca as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/2dab1da0a8024f41bca04ab69ac9b6ca 2024-12-03T09:15:00,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/2dab1da0a8024f41bca04ab69ac9b6ca, entries=150, sequenceid=119, filesize=11.7 K 2024-12-03T09:15:00,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 58a142609061c77936f842676cc6c10e in 867ms, sequenceid=119, compaction requested=true 2024-12-03T09:15:00,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:00,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:00,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:00,598 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:15:00,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:00,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:00,598 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:15:00,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:00,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:00,599 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:15:00,600 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/B is initiating minor compaction (all files) 2024-12-03T09:15:00,600 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/B in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:00,600 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/1c54e983214149f9914fe4412ee5a2fb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/ca1de98dc89c4cf9952963469f563893, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/8bd05688007d458198644e3c3581130e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/9c762d669b7b4134bcf04b09a858ed2f] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=47.0 K 2024-12-03T09:15:00,600 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:15:00,600 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c54e983214149f9914fe4412ee5a2fb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733217294103 2024-12-03T09:15:00,600 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/A is initiating minor compaction (all files) 2024-12-03T09:15:00,600 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/A in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:00,600 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/aac2417c16114c5eaf8b60af845529db, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/4ff1c63649b7446bb5e732cf478a26bd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/94cc39cdba104dc5800e23ca8b8e1e98, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/4b1f412421784ab59dfed23f9cad19ce] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=51.5 K 2024-12-03T09:15:00,601 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting ca1de98dc89c4cf9952963469f563893, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733217295252 2024-12-03T09:15:00,601 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting aac2417c16114c5eaf8b60af845529db, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733217294103 2024-12-03T09:15:00,601 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bd05688007d458198644e3c3581130e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733217297421 2024-12-03T09:15:00,601 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ff1c63649b7446bb5e732cf478a26bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733217295252 2024-12-03T09:15:00,601 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94cc39cdba104dc5800e23ca8b8e1e98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733217297421 2024-12-03T09:15:00,601 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c762d669b7b4134bcf04b09a858ed2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733217298587 2024-12-03T09:15:00,602 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b1f412421784ab59dfed23f9cad19ce, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733217298586 2024-12-03T09:15:00,611 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#B#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:00,612 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/56ee5420c833493bb6d97870d88ff0b5 is 50, key is test_row_0/B:col10/1733217299731/Put/seqid=0 2024-12-03T09:15:00,612 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#A#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:00,612 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/7b4c5aac425a4a90aa94e1e5e34832c9 is 50, key is test_row_0/A:col10/1733217299731/Put/seqid=0 2024-12-03T09:15:00,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T09:15:00,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:00,617 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:15:00,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:15:00,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:00,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:15:00,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:00,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:15:00,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T09:15:00,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/d75a6b951bcc4b34bf0968563388df9b is 50, key is test_row_0/A:col10/1733217299744/Put/seqid=0 2024-12-03T09:15:00,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742070_1246 (size=12241) 2024-12-03T09:15:00,671 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/7b4c5aac425a4a90aa94e1e5e34832c9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/7b4c5aac425a4a90aa94e1e5e34832c9 2024-12-03T09:15:00,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742069_1245 (size=12241) 2024-12-03T09:15:00,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742071_1247 (size=12001) 2024-12-03T09:15:00,676 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/d75a6b951bcc4b34bf0968563388df9b 2024-12-03T09:15:00,679 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58a142609061c77936f842676cc6c10e/A of 58a142609061c77936f842676cc6c10e into 7b4c5aac425a4a90aa94e1e5e34832c9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:00,679 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:00,679 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/A, priority=12, startTime=1733217300598; duration=0sec 2024-12-03T09:15:00,679 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:00,679 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:A 2024-12-03T09:15:00,679 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:15:00,686 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:15:00,687 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/C is initiating minor compaction (all files) 2024-12-03T09:15:00,687 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/C in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:00,687 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/21b0da63b13a473da7e2b16436e0b217, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/c3a4e6f09f5446c48ba46adac9808757, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a996d1c899eb4eea93cc0d7a332cb08d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/2dab1da0a8024f41bca04ab69ac9b6ca] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=47.0 K 2024-12-03T09:15:00,688 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21b0da63b13a473da7e2b16436e0b217, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733217294103 2024-12-03T09:15:00,690 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a4e6f09f5446c48ba46adac9808757, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733217295252 2024-12-03T09:15:00,691 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a996d1c899eb4eea93cc0d7a332cb08d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733217297421 2024-12-03T09:15:00,693 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dab1da0a8024f41bca04ab69ac9b6ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733217298587 2024-12-03T09:15:00,699 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/56ee5420c833493bb6d97870d88ff0b5 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/56ee5420c833493bb6d97870d88ff0b5 2024-12-03T09:15:00,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/f742bf6ab7794287a710d4321e1316ef is 50, key is test_row_0/B:col10/1733217299744/Put/seqid=0 2024-12-03T09:15:00,707 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58a142609061c77936f842676cc6c10e/B of 58a142609061c77936f842676cc6c10e into 56ee5420c833493bb6d97870d88ff0b5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:00,707 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:00,708 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/B, priority=12, startTime=1733217300598; duration=0sec 2024-12-03T09:15:00,708 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:00,708 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:B 2024-12-03T09:15:00,725 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#C#compaction#205 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:00,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742072_1248 (size=12001) 2024-12-03T09:15:00,726 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/3c5efa916023420fa74adfad4637b47e is 50, key is test_row_0/C:col10/1733217299731/Put/seqid=0 2024-12-03T09:15:00,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742073_1249 (size=12241) 2024-12-03T09:15:00,744 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/3c5efa916023420fa74adfad4637b47e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3c5efa916023420fa74adfad4637b47e 2024-12-03T09:15:00,750 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58a142609061c77936f842676cc6c10e/C of 58a142609061c77936f842676cc6c10e into 3c5efa916023420fa74adfad4637b47e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:00,750 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:00,750 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/C, priority=12, startTime=1733217300598; duration=0sec 2024-12-03T09:15:00,750 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:00,750 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:C 2024-12-03T09:15:00,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:00,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:00,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217300900, max timestamp=1733217300909Num uniques 859; 2024-12-03T09:15:00,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217300900, max timestamp=1733217300909Num uniques 859; 2024-12-03T09:15:00,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217300900, max timestamp=1733217300909Num uniques 859; 2024-12-03T09:15:00,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:15:00,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:15:00,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:15:00,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:00,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217360928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:00,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217360932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217360932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T09:15:01,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217361040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217361040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217361040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,126 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/f742bf6ab7794287a710d4321e1316ef 2024-12-03T09:15:01,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/0e18ab576ca8449ca7a696beae784325 is 50, key is test_row_0/C:col10/1733217299744/Put/seqid=0 2024-12-03T09:15:01,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742074_1250 (size=12001) 2024-12-03T09:15:01,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217361249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217361250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217361250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T09:15:01,544 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/0e18ab576ca8449ca7a696beae784325 2024-12-03T09:15:01,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/d75a6b951bcc4b34bf0968563388df9b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/d75a6b951bcc4b34bf0968563388df9b 2024-12-03T09:15:01,553 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/d75a6b951bcc4b34bf0968563388df9b, entries=150, sequenceid=129, filesize=11.7 K 2024-12-03T09:15:01,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/f742bf6ab7794287a710d4321e1316ef as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/f742bf6ab7794287a710d4321e1316ef 2024-12-03T09:15:01,558 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/f742bf6ab7794287a710d4321e1316ef, entries=150, sequenceid=129, filesize=11.7 K 2024-12-03T09:15:01,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/0e18ab576ca8449ca7a696beae784325 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/0e18ab576ca8449ca7a696beae784325 2024-12-03T09:15:01,563 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/0e18ab576ca8449ca7a696beae784325, entries=150, sequenceid=129, filesize=11.7 K 2024-12-03T09:15:01,564 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 58a142609061c77936f842676cc6c10e in 947ms, sequenceid=129, compaction requested=false 2024-12-03T09:15:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-03T09:15:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-03T09:15:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:01,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-12-03T09:15:01,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:15:01,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T09:15:01,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:15:01,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T09:15:01,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:15:01,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T09:15:01,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-03T09:15:01,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2530 sec 2024-12-03T09:15:01,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.2630 sec 2024-12-03T09:15:01,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217361573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217361573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217361574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/affbbef6c2404c6faca1bf6446b92b02 is 50, key is test_row_0/A:col10/1733217300931/Put/seqid=0 2024-12-03T09:15:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742075_1251 (size=14541) 2024-12-03T09:15:01,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/affbbef6c2404c6faca1bf6446b92b02 2024-12-03T09:15:01,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/3013c7d8abeb43a6a02bb735325f7296 is 50, key is test_row_0/B:col10/1733217300931/Put/seqid=0 2024-12-03T09:15:01,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742076_1252 (size=12151) 2024-12-03T09:15:01,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/3013c7d8abeb43a6a02bb735325f7296 2024-12-03T09:15:01,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/f23227f6cfda4507a7729ae1135f6193 is 50, key is test_row_0/C:col10/1733217300931/Put/seqid=0 2024-12-03T09:15:01,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742077_1253 (size=12151) 2024-12-03T09:15:01,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217361679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217361680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217361689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217361889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217361889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:01,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217361900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:02,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/f23227f6cfda4507a7729ae1135f6193 2024-12-03T09:15:02,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/affbbef6c2404c6faca1bf6446b92b02 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/affbbef6c2404c6faca1bf6446b92b02 2024-12-03T09:15:02,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/affbbef6c2404c6faca1bf6446b92b02, entries=200, sequenceid=170, filesize=14.2 K 2024-12-03T09:15:02,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/3013c7d8abeb43a6a02bb735325f7296 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/3013c7d8abeb43a6a02bb735325f7296 2024-12-03T09:15:02,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/3013c7d8abeb43a6a02bb735325f7296, entries=150, sequenceid=170, filesize=11.9 K 2024-12-03T09:15:02,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/f23227f6cfda4507a7729ae1135f6193 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/f23227f6cfda4507a7729ae1135f6193 2024-12-03T09:15:02,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/f23227f6cfda4507a7729ae1135f6193, entries=150, sequenceid=170, filesize=11.9 K 2024-12-03T09:15:02,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~415.72 KB/425696, currentSize=33.54 KB/34350 for 58a142609061c77936f842676cc6c10e in 487ms, sequenceid=170, compaction requested=true 2024-12-03T09:15:02,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:02,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:02,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:02,051 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:02,051 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:02,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:02,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:02,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:02,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:02,052 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:02,052 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:02,052 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/B is initiating minor compaction (all files) 2024-12-03T09:15:02,052 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/A is initiating minor compaction (all files) 2024-12-03T09:15:02,052 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/A in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:02,053 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/B in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:02,053 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/7b4c5aac425a4a90aa94e1e5e34832c9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/d75a6b951bcc4b34bf0968563388df9b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/affbbef6c2404c6faca1bf6446b92b02] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=37.9 K 2024-12-03T09:15:02,053 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/56ee5420c833493bb6d97870d88ff0b5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/f742bf6ab7794287a710d4321e1316ef, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/3013c7d8abeb43a6a02bb735325f7296] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=35.5 K 2024-12-03T09:15:02,053 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 56ee5420c833493bb6d97870d88ff0b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733217298587 2024-12-03T09:15:02,053 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b4c5aac425a4a90aa94e1e5e34832c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733217298587 2024-12-03T09:15:02,054 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f742bf6ab7794287a710d4321e1316ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733217299734 2024-12-03T09:15:02,054 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d75a6b951bcc4b34bf0968563388df9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733217299734 2024-12-03T09:15:02,054 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3013c7d8abeb43a6a02bb735325f7296, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733217300931 2024-12-03T09:15:02,054 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting affbbef6c2404c6faca1bf6446b92b02, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733217300919 2024-12-03T09:15:02,061 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#B#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:02,061 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/b07bba194842474694c974a66ad055a5 is 50, key is test_row_0/B:col10/1733217300931/Put/seqid=0 2024-12-03T09:15:02,061 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#A#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:02,062 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/5e08d2c26fed4f7fba4a6161e143e124 is 50, key is test_row_0/A:col10/1733217300931/Put/seqid=0 2024-12-03T09:15:02,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742079_1255 (size=12493) 2024-12-03T09:15:02,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742078_1254 (size=12493) 2024-12-03T09:15:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:02,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:15:02,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:15:02,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:02,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:15:02,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:02,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:15:02,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:02,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/5c8a0f3ded224f5fa4bc24bc76f326dc is 50, key is test_row_0/A:col10/1733217301573/Put/seqid=0 2024-12-03T09:15:02,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742080_1256 (size=16931) 2024-12-03T09:15:02,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733217302218, max timestamp=1733217302229Num uniques 859; 2024-12-03T09:15:02,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733217302218, max timestamp=1733217302229Num uniques 859; 2024-12-03T09:15:02,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733217302218, max timestamp=1733217302229Num uniques 859; 2024-12-03T09:15:02,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T09:15:02,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T09:15:02,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T09:15:02,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:02,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217362255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:02,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217362255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:02,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:02,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217362259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:02,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217362370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:02,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:02,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217362370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:02,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:02,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:02,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217362370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:02,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T09:15:02,460 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:02,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:02,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-03T09:15:02,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T09:15:02,462 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:02,462 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:02,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:02,472 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/5e08d2c26fed4f7fba4a6161e143e124 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5e08d2c26fed4f7fba4a6161e143e124 2024-12-03T09:15:02,474 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/b07bba194842474694c974a66ad055a5 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/b07bba194842474694c974a66ad055a5 2024-12-03T09:15:02,477 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/A of 58a142609061c77936f842676cc6c10e into 5e08d2c26fed4f7fba4a6161e143e124(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:02,477 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:02,477 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/A, priority=13, startTime=1733217302051; duration=0sec 2024-12-03T09:15:02,477 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:02,478 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:A 2024-12-03T09:15:02,478 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:02,479 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:02,479 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/C is initiating minor compaction (all files) 2024-12-03T09:15:02,479 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/C in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:02,479 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3c5efa916023420fa74adfad4637b47e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/0e18ab576ca8449ca7a696beae784325, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/f23227f6cfda4507a7729ae1135f6193] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=35.5 K 2024-12-03T09:15:02,480 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c5efa916023420fa74adfad4637b47e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733217298587 2024-12-03T09:15:02,480 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e18ab576ca8449ca7a696beae784325, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733217299734 2024-12-03T09:15:02,481 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f23227f6cfda4507a7729ae1135f6193, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733217300931 2024-12-03T09:15:02,481 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/B of 58a142609061c77936f842676cc6c10e into b07bba194842474694c974a66ad055a5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:02,481 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:02,481 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/B, priority=13, startTime=1733217302051; duration=0sec 2024-12-03T09:15:02,481 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:02,481 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:B 2024-12-03T09:15:02,487 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#C#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:02,488 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/8659872b482e4d9b9836b56f1213081a is 50, key is test_row_0/C:col10/1733217300931/Put/seqid=0 2024-12-03T09:15:02,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742081_1257 (size=12493) 2024-12-03T09:15:02,506 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/8659872b482e4d9b9836b56f1213081a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/8659872b482e4d9b9836b56f1213081a 2024-12-03T09:15:02,511 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/C of 58a142609061c77936f842676cc6c10e into 8659872b482e4d9b9836b56f1213081a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:02,511 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:02,511 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/C, priority=13, startTime=1733217302051; duration=0sec 2024-12-03T09:15:02,511 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:02,511 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:C 2024-12-03T09:15:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T09:15:02,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217362579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217362580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:02,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217362580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:02,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/5c8a0f3ded224f5fa4bc24bc76f326dc 2024-12-03T09:15:02,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:02,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:02,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:02,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:02,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:02,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:02,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/f3c74276f094456cbec0152f16e4cd3a is 50, key is test_row_0/B:col10/1733217301573/Put/seqid=0 2024-12-03T09:15:02,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742082_1258 (size=12151) 2024-12-03T09:15:02,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:02,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:02,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:02,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:02,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:02,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T09:15:02,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217362889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:02,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217362889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:02,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217362889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:02,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:02,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:02,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:02,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/f3c74276f094456cbec0152f16e4cd3a 2024-12-03T09:15:03,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/6a0db6b18851400fb0e6c4ca76aea48f is 50, key is test_row_0/C:col10/1733217301573/Put/seqid=0 2024-12-03T09:15:03,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742083_1259 (size=12151) 2024-12-03T09:15:03,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:03,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:03,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T09:15:03,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:03,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:03,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:03,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:03,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217363399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:03,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:03,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217363399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:03,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217363399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:03,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/6a0db6b18851400fb0e6c4ca76aea48f 2024-12-03T09:15:03,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/5c8a0f3ded224f5fa4bc24bc76f326dc as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5c8a0f3ded224f5fa4bc24bc76f326dc 2024-12-03T09:15:03,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5c8a0f3ded224f5fa4bc24bc76f326dc, entries=250, sequenceid=180, filesize=16.5 K 2024-12-03T09:15:03,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/f3c74276f094456cbec0152f16e4cd3a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/f3c74276f094456cbec0152f16e4cd3a 2024-12-03T09:15:03,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/f3c74276f094456cbec0152f16e4cd3a, entries=150, sequenceid=180, filesize=11.9 K 2024-12-03T09:15:03,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/6a0db6b18851400fb0e6c4ca76aea48f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/6a0db6b18851400fb0e6c4ca76aea48f 2024-12-03T09:15:03,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/6a0db6b18851400fb0e6c4ca76aea48f, entries=150, sequenceid=180, filesize=11.9 K 2024-12-03T09:15:03,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 58a142609061c77936f842676cc6c10e in 1268ms, sequenceid=180, compaction requested=false 2024-12-03T09:15:03,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:03,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-03T09:15:03,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:15:03,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:03,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:15:03,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:03,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:15:03,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:03,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/221cdaeaa7e74267a09e990070d6048a is 50, key is test_row_0/A:col10/1733217302254/Put/seqid=0 2024-12-03T09:15:03,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742084_1260 (size=14541) 2024-12-03T09:15:03,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217363490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:03,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217363495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:03,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:03,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:03,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T09:15:03,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217363599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:03,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217363609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:03,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:03,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:03,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217363809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:03,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:03,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217363820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:03,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:03,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:03,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/221cdaeaa7e74267a09e990070d6048a 2024-12-03T09:15:03,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/10404cd0adde486490b66243a7969810 is 50, key is test_row_0/B:col10/1733217302254/Put/seqid=0 2024-12-03T09:15:03,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742085_1261 (size=12151) 2024-12-03T09:15:03,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:03,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:03,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217364119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:04,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:04,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217364129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:04,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:04,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:04,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:04,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/10404cd0adde486490b66243a7969810 2024-12-03T09:15:04,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/3eff536e0e55419fa115cfb3a28530eb is 50, key is test_row_0/C:col10/1733217302254/Put/seqid=0 2024-12-03T09:15:04,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:04,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:04,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742086_1262 (size=12151) 2024-12-03T09:15:04,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/3eff536e0e55419fa115cfb3a28530eb 2024-12-03T09:15:04,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/221cdaeaa7e74267a09e990070d6048a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/221cdaeaa7e74267a09e990070d6048a 2024-12-03T09:15:04,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/221cdaeaa7e74267a09e990070d6048a, entries=200, sequenceid=219, filesize=14.2 K 2024-12-03T09:15:04,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/10404cd0adde486490b66243a7969810 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/10404cd0adde486490b66243a7969810 2024-12-03T09:15:04,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/10404cd0adde486490b66243a7969810, entries=150, sequenceid=219, filesize=11.9 K 2024-12-03T09:15:04,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/3eff536e0e55419fa115cfb3a28530eb as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3eff536e0e55419fa115cfb3a28530eb 2024-12-03T09:15:04,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3eff536e0e55419fa115cfb3a28530eb, entries=150, sequenceid=219, filesize=11.9 K 2024-12-03T09:15:04,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 58a142609061c77936f842676cc6c10e in 852ms, sequenceid=219, compaction requested=true 2024-12-03T09:15:04,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:04,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:04,324 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:04,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:04,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:04,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:04,324 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:04,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:04,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:04,325 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:04,325 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:04,325 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/A is initiating minor compaction (all files) 2024-12-03T09:15:04,325 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/B is initiating minor compaction (all files) 2024-12-03T09:15:04,325 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/A in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,325 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/B in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,326 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5e08d2c26fed4f7fba4a6161e143e124, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5c8a0f3ded224f5fa4bc24bc76f326dc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/221cdaeaa7e74267a09e990070d6048a] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=42.9 K 2024-12-03T09:15:04,326 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/b07bba194842474694c974a66ad055a5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/f3c74276f094456cbec0152f16e4cd3a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/10404cd0adde486490b66243a7969810] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=35.9 K 2024-12-03T09:15:04,326 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e08d2c26fed4f7fba4a6161e143e124, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733217300931 2024-12-03T09:15:04,326 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting b07bba194842474694c974a66ad055a5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733217300931 2024-12-03T09:15:04,326 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f3c74276f094456cbec0152f16e4cd3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733217301573 2024-12-03T09:15:04,326 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c8a0f3ded224f5fa4bc24bc76f326dc, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733217301570 2024-12-03T09:15:04,327 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 10404cd0adde486490b66243a7969810, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733217302253 2024-12-03T09:15:04,327 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 221cdaeaa7e74267a09e990070d6048a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733217302253 2024-12-03T09:15:04,335 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#A#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:04,335 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/8a0aee196b8e47259792962ca47f6e73 is 50, key is test_row_0/A:col10/1733217302254/Put/seqid=0 2024-12-03T09:15:04,337 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:04,338 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/e3d38e70e5fe42d486f3162191cf9b04 is 50, key is test_row_0/B:col10/1733217302254/Put/seqid=0 2024-12-03T09:15:04,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742087_1263 (size=12595) 2024-12-03T09:15:04,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742088_1264 (size=12595) 2024-12-03T09:15:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:04,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T09:15:04,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:15:04,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:04,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:15:04,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:04,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:15:04,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:04,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/7e1e2a470c4f4b62b4dac2d28b1e69c8 is 50, key is test_row_0/A:col10/1733217304411/Put/seqid=0 2024-12-03T09:15:04,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742089_1265 (size=14537) 2024-12-03T09:15:04,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:04,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:04,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217364463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:04,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217304457, max timestamp=1733217304464Num uniques 859; 2024-12-03T09:15:04,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:15:04,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217304457, max timestamp=1733217304464Num uniques 859; 2024-12-03T09:15:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:15:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:04,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217304457, max timestamp=1733217304464Num uniques 859; 2024-12-03T09:15:04,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:15:04,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217364499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:04,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217364500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:04,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217364570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:04,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:04,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:04,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:04,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217364609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:04,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217364609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T09:15:04,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217364629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:04,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:04,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217364639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:04,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:04,746 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/8a0aee196b8e47259792962ca47f6e73 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/8a0aee196b8e47259792962ca47f6e73 2024-12-03T09:15:04,751 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/e3d38e70e5fe42d486f3162191cf9b04 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/e3d38e70e5fe42d486f3162191cf9b04 2024-12-03T09:15:04,751 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/A of 58a142609061c77936f842676cc6c10e into 8a0aee196b8e47259792962ca47f6e73(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:04,751 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:04,751 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/A, priority=13, startTime=1733217304324; duration=0sec 2024-12-03T09:15:04,751 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:04,751 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:A 2024-12-03T09:15:04,751 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:04,752 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:04,753 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/C is initiating minor compaction (all files) 2024-12-03T09:15:04,753 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/C in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,753 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/8659872b482e4d9b9836b56f1213081a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/6a0db6b18851400fb0e6c4ca76aea48f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3eff536e0e55419fa115cfb3a28530eb] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=35.9 K 2024-12-03T09:15:04,753 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8659872b482e4d9b9836b56f1213081a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733217300931 2024-12-03T09:15:04,753 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a0db6b18851400fb0e6c4ca76aea48f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733217301573 2024-12-03T09:15:04,753 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eff536e0e55419fa115cfb3a28530eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733217302253 2024-12-03T09:15:04,756 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/B of 58a142609061c77936f842676cc6c10e into e3d38e70e5fe42d486f3162191cf9b04(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:04,756 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:04,756 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/B, priority=13, startTime=1733217304324; duration=0sec 2024-12-03T09:15:04,756 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:04,756 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:B 2024-12-03T09:15:04,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:04,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:04,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,760 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#C#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,760 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/cdb0c88c07d94d97a4b75d63d6e5e00e is 50, key is test_row_0/C:col10/1733217302254/Put/seqid=0 2024-12-03T09:15:04,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742090_1266 (size=12595) 2024-12-03T09:15:04,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:04,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217364779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:04,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/7e1e2a470c4f4b62b4dac2d28b1e69c8 2024-12-03T09:15:04,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:04,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217364819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:04,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217364819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:04,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/b72661c014394ad184d957fe3e2ed935 is 50, key is test_row_0/B:col10/1733217304411/Put/seqid=0 2024-12-03T09:15:04,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742091_1267 (size=9757) 2024-12-03T09:15:04,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:04,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:04,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:04,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:04,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:05,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:05,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:05,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:05,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:05,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:05,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:05,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217365089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:05,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217365129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:05,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:05,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217365129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:05,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:05,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:05,171 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/cdb0c88c07d94d97a4b75d63d6e5e00e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/cdb0c88c07d94d97a4b75d63d6e5e00e 2024-12-03T09:15:05,175 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/C of 58a142609061c77936f842676cc6c10e into cdb0c88c07d94d97a4b75d63d6e5e00e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:05,175 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:05,175 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/C, priority=13, startTime=1733217304324; duration=0sec 2024-12-03T09:15:05,175 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:05,175 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:C 2024-12-03T09:15:05,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:05,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:05,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:05,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:05,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:05,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:05,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/b72661c014394ad184d957fe3e2ed935 2024-12-03T09:15:05,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/74e4a6e212ee431993cf4294b405253a is 50, key is test_row_0/C:col10/1733217304411/Put/seqid=0 2024-12-03T09:15:05,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742092_1268 (size=9757) 2024-12-03T09:15:05,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:05,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:05,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:05,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:05,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:05,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:05,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:05,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:05,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:05,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:05,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:05,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:05,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217365599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:05,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217365639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:05,641 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:05,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217365639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217365639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:05,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/74e4a6e212ee431993cf4294b405253a 2024-12-03T09:15:05,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/7e1e2a470c4f4b62b4dac2d28b1e69c8 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/7e1e2a470c4f4b62b4dac2d28b1e69c8 2024-12-03T09:15:05,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/7e1e2a470c4f4b62b4dac2d28b1e69c8, entries=200, sequenceid=230, filesize=14.2 K 2024-12-03T09:15:05,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/b72661c014394ad184d957fe3e2ed935 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/b72661c014394ad184d957fe3e2ed935 2024-12-03T09:15:05,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/b72661c014394ad184d957fe3e2ed935, entries=100, sequenceid=230, filesize=9.5 K 2024-12-03T09:15:05,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/74e4a6e212ee431993cf4294b405253a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/74e4a6e212ee431993cf4294b405253a 2024-12-03T09:15:05,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217365659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:05,660 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:05,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:05,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/74e4a6e212ee431993cf4294b405253a, entries=100, sequenceid=230, filesize=9.5 K 2024-12-03T09:15:05,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 58a142609061c77936f842676cc6c10e in 1250ms, sequenceid=230, compaction requested=false 2024-12-03T09:15:05,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:05,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T09:15:05,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:05,676 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-03T09:15:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:15:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:15:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:15:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:05,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/3e04e04513114811b5863649d6e0d5fe is 50, key is test_row_0/A:col10/1733217304499/Put/seqid=0 2024-12-03T09:15:05,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742093_1269 (size=12301) 2024-12-03T09:15:06,086 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/3e04e04513114811b5863649d6e0d5fe 2024-12-03T09:15:06,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/29dd304410eb4576961c388def5f23f0 is 50, key is test_row_0/B:col10/1733217304499/Put/seqid=0 2024-12-03T09:15:06,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742094_1270 (size=12301) 2024-12-03T09:15:06,499 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/29dd304410eb4576961c388def5f23f0 2024-12-03T09:15:06,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/a1718bda873246fc8f5c1327de618790 is 50, key is test_row_0/C:col10/1733217304499/Put/seqid=0 2024-12-03T09:15:06,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742095_1271 (size=12301) 2024-12-03T09:15:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:06,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T09:15:06,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217366637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:06,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:06,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217366649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:06,651 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:06,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:06,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:06,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:06,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:06,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217366659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:06,661 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:06,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:06,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217366749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:06,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:06,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:06,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:06,910 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/a1718bda873246fc8f5c1327de618790 2024-12-03T09:15:06,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/3e04e04513114811b5863649d6e0d5fe as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/3e04e04513114811b5863649d6e0d5fe 2024-12-03T09:15:06,918 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/3e04e04513114811b5863649d6e0d5fe, entries=150, sequenceid=268, filesize=12.0 K 2024-12-03T09:15:06,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/29dd304410eb4576961c388def5f23f0 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/29dd304410eb4576961c388def5f23f0 2024-12-03T09:15:06,923 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/29dd304410eb4576961c388def5f23f0, entries=150, sequenceid=268, filesize=12.0 K 2024-12-03T09:15:06,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/a1718bda873246fc8f5c1327de618790 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a1718bda873246fc8f5c1327de618790 2024-12-03T09:15:06,928 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a1718bda873246fc8f5c1327de618790, entries=150, sequenceid=268, filesize=12.0 K 2024-12-03T09:15:06,928 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 58a142609061c77936f842676cc6c10e in 1252ms, sequenceid=268, compaction requested=true 2024-12-03T09:15:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:06,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-03T09:15:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-03T09:15:06,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-03T09:15:06,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4680 sec 2024-12-03T09:15:06,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 4.4720 sec 2024-12-03T09:15:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:06,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T09:15:06,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:15:06,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:06,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:15:06,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:06,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:15:06,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:06,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/218a751a963a4848b334186eaa2da8b6 is 50, key is test_row_0/A:col10/1733217306623/Put/seqid=0 2024-12-03T09:15:06,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742096_1272 (size=14741) 2024-12-03T09:15:07,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217367048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:07,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:07,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:07,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:07,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217367160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:07,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:07,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:07,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:07,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/218a751a963a4848b334186eaa2da8b6 2024-12-03T09:15:07,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217367370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:07,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/20d3f72081eb41779b5c89187f6abcf1 is 50, key is test_row_0/B:col10/1733217306623/Put/seqid=0 2024-12-03T09:15:07,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742097_1273 (size=12301) 2024-12-03T09:15:07,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217367650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:07,652 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:07,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:07,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217367669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:07,674 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:07,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:07,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217367679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:07,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/20d3f72081eb41779b5c89187f6abcf1 2024-12-03T09:15:07,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/f81f13cba49d41a19a16d5b7bbc82667 is 50, key is test_row_0/C:col10/1733217306623/Put/seqid=0 2024-12-03T09:15:07,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742098_1274 (size=12301) 2024-12-03T09:15:08,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:08,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217368189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:08,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:08,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:08,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:08,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/f81f13cba49d41a19a16d5b7bbc82667 2024-12-03T09:15:08,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/218a751a963a4848b334186eaa2da8b6 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/218a751a963a4848b334186eaa2da8b6 2024-12-03T09:15:08,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/218a751a963a4848b334186eaa2da8b6, entries=200, sequenceid=280, filesize=14.4 K 2024-12-03T09:15:08,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/20d3f72081eb41779b5c89187f6abcf1 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/20d3f72081eb41779b5c89187f6abcf1 2024-12-03T09:15:08,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/20d3f72081eb41779b5c89187f6abcf1, entries=150, sequenceid=280, filesize=12.0 K 2024-12-03T09:15:08,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/f81f13cba49d41a19a16d5b7bbc82667 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/f81f13cba49d41a19a16d5b7bbc82667 2024-12-03T09:15:08,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/f81f13cba49d41a19a16d5b7bbc82667, entries=150, sequenceid=280, filesize=12.0 K 2024-12-03T09:15:08,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 58a142609061c77936f842676cc6c10e in 1271ms, sequenceid=280, compaction requested=true 2024-12-03T09:15:08,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:08,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:08,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:08,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:08,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:08,232 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:15:08,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:08,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:08,232 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:15:08,233 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:15:08,233 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/B is initiating minor compaction (all files) 2024-12-03T09:15:08,234 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/B in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:08,234 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/e3d38e70e5fe42d486f3162191cf9b04, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/b72661c014394ad184d957fe3e2ed935, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/29dd304410eb4576961c388def5f23f0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/20d3f72081eb41779b5c89187f6abcf1] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=45.9 K 2024-12-03T09:15:08,234 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54174 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:15:08,234 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/A is initiating minor compaction (all files) 2024-12-03T09:15:08,234 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/A in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:08,234 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/8a0aee196b8e47259792962ca47f6e73, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/7e1e2a470c4f4b62b4dac2d28b1e69c8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/3e04e04513114811b5863649d6e0d5fe, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/218a751a963a4848b334186eaa2da8b6] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=52.9 K 2024-12-03T09:15:08,234 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting e3d38e70e5fe42d486f3162191cf9b04, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733217302253 2024-12-03T09:15:08,234 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a0aee196b8e47259792962ca47f6e73, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733217302253 2024-12-03T09:15:08,234 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting b72661c014394ad184d957fe3e2ed935, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733217303493 2024-12-03T09:15:08,235 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 29dd304410eb4576961c388def5f23f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733217304492 2024-12-03T09:15:08,235 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e1e2a470c4f4b62b4dac2d28b1e69c8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733217303489 2024-12-03T09:15:08,235 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 20d3f72081eb41779b5c89187f6abcf1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733217306623 2024-12-03T09:15:08,235 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e04e04513114811b5863649d6e0d5fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733217304492 2024-12-03T09:15:08,235 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 218a751a963a4848b334186eaa2da8b6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733217306623 2024-12-03T09:15:08,243 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:08,243 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/7acaaa46a6b84b8e987f0d8013877de8 is 50, key is test_row_0/B:col10/1733217306623/Put/seqid=0 2024-12-03T09:15:08,251 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#A#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:08,251 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/ffc1e9f5c2d3467fb6f43b67277c4339 is 50, key is test_row_0/A:col10/1733217306623/Put/seqid=0 2024-12-03T09:15:08,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742099_1275 (size=12881) 2024-12-03T09:15:08,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742100_1276 (size=12881) 2024-12-03T09:15:08,261 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/ffc1e9f5c2d3467fb6f43b67277c4339 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/ffc1e9f5c2d3467fb6f43b67277c4339 2024-12-03T09:15:08,268 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58a142609061c77936f842676cc6c10e/A of 58a142609061c77936f842676cc6c10e into ffc1e9f5c2d3467fb6f43b67277c4339(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:08,268 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:08,268 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/A, priority=12, startTime=1733217308232; duration=0sec 2024-12-03T09:15:08,268 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:08,268 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:A 2024-12-03T09:15:08,268 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:15:08,269 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:15:08,269 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/C is initiating minor compaction (all files) 2024-12-03T09:15:08,269 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/C in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:08,269 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/cdb0c88c07d94d97a4b75d63d6e5e00e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/74e4a6e212ee431993cf4294b405253a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a1718bda873246fc8f5c1327de618790, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/f81f13cba49d41a19a16d5b7bbc82667] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=45.9 K 2024-12-03T09:15:08,270 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdb0c88c07d94d97a4b75d63d6e5e00e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733217302253 2024-12-03T09:15:08,270 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74e4a6e212ee431993cf4294b405253a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733217303493 2024-12-03T09:15:08,270 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1718bda873246fc8f5c1327de618790, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733217304492 2024-12-03T09:15:08,271 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f81f13cba49d41a19a16d5b7bbc82667, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733217306623 2024-12-03T09:15:08,280 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:08,281 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/6edb022f55b649f0839003dac847b710 is 50, key is test_row_0/C:col10/1733217306623/Put/seqid=0 2024-12-03T09:15:08,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742101_1277 (size=12881) 2024-12-03T09:15:08,295 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/6edb022f55b649f0839003dac847b710 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/6edb022f55b649f0839003dac847b710 2024-12-03T09:15:08,302 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58a142609061c77936f842676cc6c10e/C of 58a142609061c77936f842676cc6c10e into 6edb022f55b649f0839003dac847b710(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:08,302 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:08,302 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/C, priority=12, startTime=1733217308232; duration=0sec 2024-12-03T09:15:08,303 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:08,303 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:C 2024-12-03T09:15:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:08,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T09:15:08,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:15:08,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:08,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:15:08,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:08,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:15:08,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:08,667 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/7acaaa46a6b84b8e987f0d8013877de8 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/7acaaa46a6b84b8e987f0d8013877de8 2024-12-03T09:15:08,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/f577ee5d41e8476d8a61f0870d983363 is 50, key is test_row_0/A:col10/1733217307038/Put/seqid=0 2024-12-03T09:15:08,684 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58a142609061c77936f842676cc6c10e/B of 58a142609061c77936f842676cc6c10e into 7acaaa46a6b84b8e987f0d8013877de8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:08,684 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:08,685 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/B, priority=12, startTime=1733217308232; duration=0sec 2024-12-03T09:15:08,685 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:08,685 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:B 2024-12-03T09:15:08,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217368697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:08,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:08,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217368701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:08,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:08,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742102_1278 (size=12301) 2024-12-03T09:15:08,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217368809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:08,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:08,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:08,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217368809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:08,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:08,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:08,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:08,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:09,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217369019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:09,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217369019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:09,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:09,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/f577ee5d41e8476d8a61f0870d983363 2024-12-03T09:15:09,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/9bea2e0a18f244c383574576172944cf is 50, key is test_row_0/B:col10/1733217307038/Put/seqid=0 2024-12-03T09:15:09,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742103_1279 (size=12301) 2024-12-03T09:15:09,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/9bea2e0a18f244c383574576172944cf 2024-12-03T09:15:09,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/3f1c9ff315d24e89a7815d816a656ece is 50, key is test_row_0/C:col10/1733217307038/Put/seqid=0 2024-12-03T09:15:09,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742104_1280 (size=12301) 2024-12-03T09:15:09,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217369200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:09,201 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:09,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217369330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:09,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:09,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217369330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:09,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/3f1c9ff315d24e89a7815d816a656ece 2024-12-03T09:15:09,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/f577ee5d41e8476d8a61f0870d983363 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/f577ee5d41e8476d8a61f0870d983363 2024-12-03T09:15:09,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/f577ee5d41e8476d8a61f0870d983363, entries=150, sequenceid=303, filesize=12.0 K 2024-12-03T09:15:09,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/9bea2e0a18f244c383574576172944cf as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/9bea2e0a18f244c383574576172944cf 2024-12-03T09:15:09,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/9bea2e0a18f244c383574576172944cf, entries=150, sequenceid=303, filesize=12.0 K 2024-12-03T09:15:09,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/3f1c9ff315d24e89a7815d816a656ece as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3f1c9ff315d24e89a7815d816a656ece 2024-12-03T09:15:09,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3f1c9ff315d24e89a7815d816a656ece, entries=150, sequenceid=303, filesize=12.0 K 2024-12-03T09:15:09,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 58a142609061c77936f842676cc6c10e in 898ms, sequenceid=303, compaction requested=false 2024-12-03T09:15:09,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:09,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T09:15:09,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:15:09,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:09,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:15:09,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:09,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:15:09,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:09,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/a409085dd1764b9986c50ba6c00455bc is 50, key is test_row_0/A:col10/1733217308696/Put/seqid=0 2024-12-03T09:15:09,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742105_1281 (size=14741) 2024-12-03T09:15:09,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217369868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:09,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:09,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217369871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:09,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:09,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217369979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217369979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:09,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:10,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217370190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:10,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217370190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:10,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:10,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/a409085dd1764b9986c50ba6c00455bc 2024-12-03T09:15:10,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/08bba8a2adf94803b7b3411939ae4701 is 50, key is test_row_0/B:col10/1733217308696/Put/seqid=0 2024-12-03T09:15:10,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742106_1282 (size=12301) 2024-12-03T09:15:10,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217370499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:10,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:10,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:10,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217370499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:10,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:10,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:10,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:10,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:10,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/08bba8a2adf94803b7b3411939ae4701 2024-12-03T09:15:10,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/9751512598524c21813347340df510fc is 50, key is test_row_0/C:col10/1733217308696/Put/seqid=0 2024-12-03T09:15:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T09:15:10,679 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:10,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-03T09:15:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T09:15:10,682 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:10,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742107_1283 (size=12301) 2024-12-03T09:15:10,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/9751512598524c21813347340df510fc 2024-12-03T09:15:10,683 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:10,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:10,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/a409085dd1764b9986c50ba6c00455bc as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/a409085dd1764b9986c50ba6c00455bc 2024-12-03T09:15:10,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/a409085dd1764b9986c50ba6c00455bc, entries=200, sequenceid=318, filesize=14.4 K 2024-12-03T09:15:10,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/08bba8a2adf94803b7b3411939ae4701 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/08bba8a2adf94803b7b3411939ae4701 2024-12-03T09:15:10,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/08bba8a2adf94803b7b3411939ae4701, entries=150, sequenceid=318, filesize=12.0 K 2024-12-03T09:15:10,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/9751512598524c21813347340df510fc as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/9751512598524c21813347340df510fc 2024-12-03T09:15:10,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/9751512598524c21813347340df510fc, entries=150, sequenceid=318, filesize=12.0 K 2024-12-03T09:15:10,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 58a142609061c77936f842676cc6c10e in 865ms, sequenceid=318, compaction requested=true 2024-12-03T09:15:10,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:10,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:10,706 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:10,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:10,706 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:10,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:10,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:10,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:10,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:10,707 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:10,707 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:10,707 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/B is initiating minor compaction (all files) 2024-12-03T09:15:10,707 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/A is initiating minor compaction (all files) 2024-12-03T09:15:10,707 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/B in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:10,707 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/A in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:10,707 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/ffc1e9f5c2d3467fb6f43b67277c4339, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/f577ee5d41e8476d8a61f0870d983363, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/a409085dd1764b9986c50ba6c00455bc] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=39.0 K 2024-12-03T09:15:10,707 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/7acaaa46a6b84b8e987f0d8013877de8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/9bea2e0a18f244c383574576172944cf, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/08bba8a2adf94803b7b3411939ae4701] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=36.6 K 2024-12-03T09:15:10,707 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7acaaa46a6b84b8e987f0d8013877de8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733217306623 2024-12-03T09:15:10,707 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc1e9f5c2d3467fb6f43b67277c4339, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733217306623 2024-12-03T09:15:10,707 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bea2e0a18f244c383574576172944cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733217307038 2024-12-03T09:15:10,708 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f577ee5d41e8476d8a61f0870d983363, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733217307038 2024-12-03T09:15:10,708 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 08bba8a2adf94803b7b3411939ae4701, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733217308690 2024-12-03T09:15:10,708 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a409085dd1764b9986c50ba6c00455bc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733217308690 2024-12-03T09:15:10,715 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:10,716 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/94cdecddf77e417082179ba674b0bea9 is 50, key is test_row_0/A:col10/1733217308696/Put/seqid=0 2024-12-03T09:15:10,730 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:10,731 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/88946c3a72a4449b9076a5661cd3db4d is 50, key is test_row_0/B:col10/1733217308696/Put/seqid=0 2024-12-03T09:15:10,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742108_1284 (size=12983) 2024-12-03T09:15:10,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742109_1285 (size=12983) 2024-12-03T09:15:10,758 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/88946c3a72a4449b9076a5661cd3db4d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/88946c3a72a4449b9076a5661cd3db4d 2024-12-03T09:15:10,765 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/B of 58a142609061c77936f842676cc6c10e into 88946c3a72a4449b9076a5661cd3db4d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:10,765 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:10,766 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/B, priority=13, startTime=1733217310706; duration=0sec 2024-12-03T09:15:10,766 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:10,766 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:B 2024-12-03T09:15:10,766 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:10,767 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:10,767 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/C is initiating minor compaction (all files) 2024-12-03T09:15:10,768 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/C in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:10,768 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/6edb022f55b649f0839003dac847b710, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3f1c9ff315d24e89a7815d816a656ece, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/9751512598524c21813347340df510fc] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=36.6 K 2024-12-03T09:15:10,768 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6edb022f55b649f0839003dac847b710, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733217306623 2024-12-03T09:15:10,768 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f1c9ff315d24e89a7815d816a656ece, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733217307038 2024-12-03T09:15:10,769 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9751512598524c21813347340df510fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733217308690 2024-12-03T09:15:10,781 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:10,782 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/ec2f5f83c725440585e3279d657b0660 is 50, key is test_row_0/C:col10/1733217308696/Put/seqid=0 2024-12-03T09:15:10,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742110_1286 (size=12983) 2024-12-03T09:15:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T09:15:10,794 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/ec2f5f83c725440585e3279d657b0660 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/ec2f5f83c725440585e3279d657b0660 2024-12-03T09:15:10,801 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/C of 58a142609061c77936f842676cc6c10e into ec2f5f83c725440585e3279d657b0660(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:10,801 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:10,802 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/C, priority=13, startTime=1733217310706; duration=0sec 2024-12-03T09:15:10,802 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:10,802 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:C 2024-12-03T09:15:10,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-03T09:15:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:10,837 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T09:15:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:15:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:15:10,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:10,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:15:10,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:10,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/8ac045d6058b4f9a8d3ec001fbd892b4 is 50, key is test_row_0/A:col10/1733217309863/Put/seqid=0 2024-12-03T09:15:10,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742111_1287 (size=12301) 2024-12-03T09:15:10,854 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/8ac045d6058b4f9a8d3ec001fbd892b4 2024-12-03T09:15:10,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/cbe966d11bc1461ebf8dc2bd9f469ede is 50, key is test_row_0/B:col10/1733217309863/Put/seqid=0 2024-12-03T09:15:10,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742112_1288 (size=12301) 2024-12-03T09:15:10,871 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/cbe966d11bc1461ebf8dc2bd9f469ede 2024-12-03T09:15:10,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/c9e9697e48ac4939945131cc3f204e60 is 50, key is test_row_0/C:col10/1733217309863/Put/seqid=0 2024-12-03T09:15:10,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742113_1289 (size=12301) 2024-12-03T09:15:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T09:15:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:11,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:11,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217371032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:11,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217371036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:11,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:11,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217371139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:11,149 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/94cdecddf77e417082179ba674b0bea9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/94cdecddf77e417082179ba674b0bea9 2024-12-03T09:15:11,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217371149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:11,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:11,153 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/A of 58a142609061c77936f842676cc6c10e into 94cdecddf77e417082179ba674b0bea9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:11,153 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:11,153 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/A, priority=13, startTime=1733217310706; duration=0sec 2024-12-03T09:15:11,153 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:11,153 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:A 2024-12-03T09:15:11,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47930 deadline: 1733217371220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:11,221 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:11,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:11,287 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/c9e9697e48ac4939945131cc3f204e60 2024-12-03T09:15:11,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/8ac045d6058b4f9a8d3ec001fbd892b4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/8ac045d6058b4f9a8d3ec001fbd892b4 2024-12-03T09:15:11,294 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/8ac045d6058b4f9a8d3ec001fbd892b4, entries=150, sequenceid=338, filesize=12.0 K 2024-12-03T09:15:11,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/cbe966d11bc1461ebf8dc2bd9f469ede as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/cbe966d11bc1461ebf8dc2bd9f469ede 2024-12-03T09:15:11,299 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/cbe966d11bc1461ebf8dc2bd9f469ede, entries=150, sequenceid=338, filesize=12.0 K 2024-12-03T09:15:11,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/c9e9697e48ac4939945131cc3f204e60 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/c9e9697e48ac4939945131cc3f204e60 2024-12-03T09:15:11,305 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/c9e9697e48ac4939945131cc3f204e60, entries=150, sequenceid=338, filesize=12.0 K 2024-12-03T09:15:11,306 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 58a142609061c77936f842676cc6c10e in 469ms, sequenceid=338, compaction requested=false 2024-12-03T09:15:11,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:11,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:11,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-03T09:15:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-03T09:15:11,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-03T09:15:11,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 623 msec 2024-12-03T09:15:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T09:15:11,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 630 msec 2024-12-03T09:15:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:11,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T09:15:11,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:15:11,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:11,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:15:11,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:11,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:15:11,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:11,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/fb9d3a7137f145b4894a3084015b755b is 50, key is test_row_0/A:col10/1733217311350/Put/seqid=0 2024-12-03T09:15:11,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742114_1290 (size=14741) 2024-12-03T09:15:11,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217371380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:11,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217371383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:11,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:11,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217371489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:11,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217371489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:11,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47970 deadline: 1733217371680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:11,681 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:11,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47994 deadline: 1733217371689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:11,691 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 58a142609061c77936f842676cc6c10e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:11,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217371699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217371699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:11,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/fb9d3a7137f145b4894a3084015b755b 2024-12-03T09:15:11,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/8d1ee8b987c94e13b55325f5489e7a5d is 50, key is test_row_0/B:col10/1733217311350/Put/seqid=0 2024-12-03T09:15:11,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742115_1291 (size=12301) 2024-12-03T09:15:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T09:15:11,820 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:11,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-03T09:15:11,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T09:15:11,822 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:11,822 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:11,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T09:15:11,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T09:15:11,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:11,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:11,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:11,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:11,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:11,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:12,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217372009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:12,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217372009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:12,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T09:15:12,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:12,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:12,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:12,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:12,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T09:15:12,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/8d1ee8b987c94e13b55325f5489e7a5d 2024-12-03T09:15:12,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/a004de19181246789f6f3ff0a7164e23 is 50, key is test_row_0/C:col10/1733217311350/Put/seqid=0 2024-12-03T09:15:12,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742116_1292 (size=12301) 2024-12-03T09:15:12,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T09:15:12,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:12,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:12,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:12,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:12,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:12,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T09:15:12,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:12,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:12,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:12,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:12,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T09:15:12,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:12,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47918 deadline: 1733217372519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733217372519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:12,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/a004de19181246789f6f3ff0a7164e23 2024-12-03T09:15:12,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T09:15:12,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:12,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:12,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:12,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:12,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:12,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/fb9d3a7137f145b4894a3084015b755b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/fb9d3a7137f145b4894a3084015b755b 2024-12-03T09:15:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:12,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/fb9d3a7137f145b4894a3084015b755b, entries=200, sequenceid=354, filesize=14.4 K 2024-12-03T09:15:12,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/8d1ee8b987c94e13b55325f5489e7a5d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/8d1ee8b987c94e13b55325f5489e7a5d 2024-12-03T09:15:12,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/8d1ee8b987c94e13b55325f5489e7a5d, entries=150, sequenceid=354, filesize=12.0 K 2024-12-03T09:15:12,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/a004de19181246789f6f3ff0a7164e23 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a004de19181246789f6f3ff0a7164e23 2024-12-03T09:15:12,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a004de19181246789f6f3ff0a7164e23, entries=150, sequenceid=354, filesize=12.0 K 2024-12-03T09:15:12,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 58a142609061c77936f842676cc6c10e in 1246ms, sequenceid=354, compaction requested=true 2024-12-03T09:15:12,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:12,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:12,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:12,598 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:12,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:12,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:12,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58a142609061c77936f842676cc6c10e:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:12,598 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:12,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:12,599 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:12,599 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:12,599 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/B is initiating minor compaction (all files) 2024-12-03T09:15:12,599 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/A is initiating minor compaction (all files) 2024-12-03T09:15:12,599 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/A in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:12,599 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/B in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:12,599 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/88946c3a72a4449b9076a5661cd3db4d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/cbe966d11bc1461ebf8dc2bd9f469ede, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/8d1ee8b987c94e13b55325f5489e7a5d] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=36.7 K 2024-12-03T09:15:12,599 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/94cdecddf77e417082179ba674b0bea9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/8ac045d6058b4f9a8d3ec001fbd892b4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/fb9d3a7137f145b4894a3084015b755b] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=39.1 K 2024-12-03T09:15:12,599 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 88946c3a72a4449b9076a5661cd3db4d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733217308690 2024-12-03T09:15:12,599 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94cdecddf77e417082179ba674b0bea9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733217308690 2024-12-03T09:15:12,600 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting cbe966d11bc1461ebf8dc2bd9f469ede, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733217309863 2024-12-03T09:15:12,600 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ac045d6058b4f9a8d3ec001fbd892b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733217309863 2024-12-03T09:15:12,600 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d1ee8b987c94e13b55325f5489e7a5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733217311029 2024-12-03T09:15:12,600 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb9d3a7137f145b4894a3084015b755b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733217311029 2024-12-03T09:15:12,607 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:12,607 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/6e1cd705328c4bc1841f0743061ed080 is 50, key is test_row_0/B:col10/1733217311350/Put/seqid=0 2024-12-03T09:15:12,610 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:12,611 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/a1609042a98942749e6cd91e63b7223b is 50, key is test_row_0/A:col10/1733217311350/Put/seqid=0 2024-12-03T09:15:12,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742117_1293 (size=13085) 2024-12-03T09:15:12,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742118_1294 (size=13085) 2024-12-03T09:15:12,625 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/6e1cd705328c4bc1841f0743061ed080 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/6e1cd705328c4bc1841f0743061ed080 2024-12-03T09:15:12,626 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/a1609042a98942749e6cd91e63b7223b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/a1609042a98942749e6cd91e63b7223b 2024-12-03T09:15:12,631 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/B of 58a142609061c77936f842676cc6c10e into 6e1cd705328c4bc1841f0743061ed080(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:12,631 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:12,631 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/A of 58a142609061c77936f842676cc6c10e into a1609042a98942749e6cd91e63b7223b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:12,631 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/B, priority=13, startTime=1733217312598; duration=0sec 2024-12-03T09:15:12,631 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:12,631 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/A, priority=13, startTime=1733217312598; duration=0sec 2024-12-03T09:15:12,631 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:12,631 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:B 2024-12-03T09:15:12,631 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:12,632 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:12,632 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:A 2024-12-03T09:15:12,633 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:12,633 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 58a142609061c77936f842676cc6c10e/C is initiating minor compaction (all files) 2024-12-03T09:15:12,633 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58a142609061c77936f842676cc6c10e/C in TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:12,633 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/ec2f5f83c725440585e3279d657b0660, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/c9e9697e48ac4939945131cc3f204e60, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a004de19181246789f6f3ff0a7164e23] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp, totalSize=36.7 K 2024-12-03T09:15:12,633 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting ec2f5f83c725440585e3279d657b0660, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1733217308690 2024-12-03T09:15:12,634 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting c9e9697e48ac4939945131cc3f204e60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733217309863 2024-12-03T09:15:12,634 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting a004de19181246789f6f3ff0a7164e23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1733217311029 2024-12-03T09:15:12,642 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58a142609061c77936f842676cc6c10e#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:12,643 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/e5cf1494f8ce41d38bdc916aaa21295c is 50, key is test_row_0/C:col10/1733217311350/Put/seqid=0 2024-12-03T09:15:12,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742119_1295 (size=13085) 2024-12-03T09:15:12,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-03T09:15:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:12,737 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T09:15:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:15:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:15:12,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:12,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:15:12,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:12,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/82547a29af784130a89a1bd7e3bfa541 is 50, key is test_row_0/A:col10/1733217311372/Put/seqid=0 2024-12-03T09:15:12,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742120_1296 (size=12301) 2024-12-03T09:15:12,799 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T09:15:12,856 INFO [Thread-1066 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1066. 2024-12-03T09:15:12,856 DEBUG [Thread-1066 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:12,857 DEBUG [Thread-1066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:12,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:12,857 INFO [Thread-1068 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1068. 2024-12-03T09:15:12,857 DEBUG [Thread-1068 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:12,857 DEBUG [Thread-1066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:12,857 DEBUG [Thread-1068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:12,857 DEBUG [Thread-1068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:12,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:12,857 INFO [Thread-1064 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1064. 2024-12-03T09:15:12,857 DEBUG [Thread-1064 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:12,857 DEBUG [Thread-1064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:12,857 DEBUG [Thread-1064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:12,858 INFO [Thread-1062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1062. 2024-12-03T09:15:12,858 DEBUG [Thread-1062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:12,858 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:12,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:12,858 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:12,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:12,859 INFO [Thread-1070 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1070. 2024-12-03T09:15:12,859 DEBUG [Thread-1070 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:12,859 DEBUG [Thread-1070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:12,859 DEBUG [Thread-1070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:12,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T09:15:13,059 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/e5cf1494f8ce41d38bdc916aaa21295c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/e5cf1494f8ce41d38bdc916aaa21295c 2024-12-03T09:15:13,065 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58a142609061c77936f842676cc6c10e/C of 58a142609061c77936f842676cc6c10e into e5cf1494f8ce41d38bdc916aaa21295c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:13,065 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:13,065 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., storeName=58a142609061c77936f842676cc6c10e/C, priority=13, startTime=1733217312598; duration=0sec 2024-12-03T09:15:13,066 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:13,066 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58a142609061c77936f842676cc6c10e:C 2024-12-03T09:15:13,146 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/82547a29af784130a89a1bd7e3bfa541 2024-12-03T09:15:13,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/4f3c40d2ca36401c853da7cb0c74674d is 50, key is test_row_0/B:col10/1733217311372/Put/seqid=0 2024-12-03T09:15:13,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742121_1297 (size=12301) 2024-12-03T09:15:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:13,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. as already flushing 2024-12-03T09:15:13,534 INFO [Thread-1057 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1057. 2024-12-03T09:15:13,534 DEBUG [Thread-1057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:13,534 DEBUG [Thread-1057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:13,535 DEBUG [Thread-1057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:13,535 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:13,543 INFO [Thread-1055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1055. 2024-12-03T09:15:13,543 DEBUG [Thread-1055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:13,543 DEBUG [Thread-1055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:13,543 DEBUG [Thread-1055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:13,543 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:13,598 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/4f3c40d2ca36401c853da7cb0c74674d 2024-12-03T09:15:13,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/4d2b891ac95b45f8923c063fb7a8f3bf is 50, key is test_row_0/C:col10/1733217311372/Put/seqid=0 2024-12-03T09:15:13,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742122_1298 (size=12301) 2024-12-03T09:15:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T09:15:14,016 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/4d2b891ac95b45f8923c063fb7a8f3bf 2024-12-03T09:15:14,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/82547a29af784130a89a1bd7e3bfa541 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/82547a29af784130a89a1bd7e3bfa541 2024-12-03T09:15:14,027 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/82547a29af784130a89a1bd7e3bfa541, entries=150, sequenceid=373, filesize=12.0 K 2024-12-03T09:15:14,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/4f3c40d2ca36401c853da7cb0c74674d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/4f3c40d2ca36401c853da7cb0c74674d 2024-12-03T09:15:14,033 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/4f3c40d2ca36401c853da7cb0c74674d, entries=150, sequenceid=373, filesize=12.0 K 2024-12-03T09:15:14,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/4d2b891ac95b45f8923c063fb7a8f3bf as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/4d2b891ac95b45f8923c063fb7a8f3bf 2024-12-03T09:15:14,037 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/4d2b891ac95b45f8923c063fb7a8f3bf, entries=150, sequenceid=373, filesize=12.0 K 2024-12-03T09:15:14,038 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=13.42 KB/13740 for 58a142609061c77936f842676cc6c10e in 1301ms, sequenceid=373, compaction requested=false 2024-12-03T09:15:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 58a142609061c77936f842676cc6c10e: 2024-12-03T09:15:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-03T09:15:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-03T09:15:14,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-03T09:15:14,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2170 sec 2024-12-03T09:15:14,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.2210 sec 2024-12-03T09:15:15,254 INFO [Thread-1059 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1059. 2024-12-03T09:15:15,254 DEBUG [Thread-1059 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:15,254 DEBUG [Thread-1059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:15,254 DEBUG [Thread-1059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:15,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-03T09:15:16,001 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:21,735 INFO [Thread-1051 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1051. 2024-12-03T09:15:21,735 DEBUG [Thread-1051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:21,735 DEBUG [Thread-1051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:21,736 DEBUG [Thread-1051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:21,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:21,793 INFO [Thread-1053 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1053. 2024-12-03T09:15:21,793 DEBUG [Thread-1053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:21,793 DEBUG [Thread-1053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:21,794 DEBUG [Thread-1053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:21,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:21,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T09:15:21,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-12-03T09:15:21,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-03T09:15:21,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-12-03T09:15:21,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-03T09:15:21,794 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-12-03T09:15:21,795 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T09:15:21,795 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4273 2024-12-03T09:15:21,795 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4180 2024-12-03T09:15:21,795 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4250 2024-12-03T09:15:21,795 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4274 2024-12-03T09:15:21,795 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4187 2024-12-03T09:15:21,795 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T09:15:21,795 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T09:15:21,795 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:15:21,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:21,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:21,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:21,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T09:15:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T09:15:21,804 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217321804"}]},"ts":"1733217321804"} 2024-12-03T09:15:21,805 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T09:15:21,805 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T09:15:21,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T09:15:21,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58a142609061c77936f842676cc6c10e, UNASSIGN}] 2024-12-03T09:15:21,808 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58a142609061c77936f842676cc6c10e, UNASSIGN 2024-12-03T09:15:21,808 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=58a142609061c77936f842676cc6c10e, regionState=CLOSING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:21,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58a142609061c77936f842676cc6c10e, UNASSIGN because future has completed 2024-12-03T09:15:21,810 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T09:15:21,810 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:15:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T09:15:21,964 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:21,964 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T09:15:21,964 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 58a142609061c77936f842676cc6c10e, disabling compactions & flushes 2024-12-03T09:15:21,964 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:21,964 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:21,964 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. after waiting 0 ms 2024-12-03T09:15:21,965 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:21,965 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 58a142609061c77936f842676cc6c10e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T09:15:21,966 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=A 2024-12-03T09:15:21,966 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:21,966 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=B 2024-12-03T09:15:21,966 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:21,966 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 58a142609061c77936f842676cc6c10e, store=C 2024-12-03T09:15:21,966 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:21,972 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/a389be1d003e4a4faddb3e79f21de33c is 50, key is test_row_0/A:col10/1733217321790/Put/seqid=0 2024-12-03T09:15:21,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742123_1299 (size=12301) 2024-12-03T09:15:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T09:15:22,377 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/a389be1d003e4a4faddb3e79f21de33c 2024-12-03T09:15:22,391 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/a9834175765545a5aa89c966754a0f7a is 50, key is test_row_0/B:col10/1733217321790/Put/seqid=0 2024-12-03T09:15:22,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742124_1300 (size=12301) 2024-12-03T09:15:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T09:15:22,796 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/a9834175765545a5aa89c966754a0f7a 2024-12-03T09:15:22,809 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/68d4f134cbf841a78ca0fe675b57b914 is 50, key is test_row_0/C:col10/1733217321790/Put/seqid=0 2024-12-03T09:15:22,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742125_1301 (size=12301) 2024-12-03T09:15:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T09:15:23,216 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/68d4f134cbf841a78ca0fe675b57b914 2024-12-03T09:15:23,224 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/A/a389be1d003e4a4faddb3e79f21de33c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/a389be1d003e4a4faddb3e79f21de33c 2024-12-03T09:15:23,230 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/a389be1d003e4a4faddb3e79f21de33c, entries=150, sequenceid=382, filesize=12.0 K 2024-12-03T09:15:23,231 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/B/a9834175765545a5aa89c966754a0f7a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/a9834175765545a5aa89c966754a0f7a 2024-12-03T09:15:23,236 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/a9834175765545a5aa89c966754a0f7a, entries=150, sequenceid=382, filesize=12.0 K 2024-12-03T09:15:23,236 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/.tmp/C/68d4f134cbf841a78ca0fe675b57b914 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/68d4f134cbf841a78ca0fe675b57b914 2024-12-03T09:15:23,240 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/68d4f134cbf841a78ca0fe675b57b914, entries=150, sequenceid=382, filesize=12.0 K 2024-12-03T09:15:23,241 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 58a142609061c77936f842676cc6c10e in 1276ms, sequenceid=382, compaction requested=true 2024-12-03T09:15:23,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/017b709515334f99956447b9b40b8c8a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5cb899849ff24b8b8fa4f449f7f488b6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/aac2417c16114c5eaf8b60af845529db, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/c95c3fe4a6c14a9ba4f6fee1524fa5ca, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/4ff1c63649b7446bb5e732cf478a26bd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/94cc39cdba104dc5800e23ca8b8e1e98, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/4b1f412421784ab59dfed23f9cad19ce, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/7b4c5aac425a4a90aa94e1e5e34832c9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/d75a6b951bcc4b34bf0968563388df9b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/affbbef6c2404c6faca1bf6446b92b02, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5e08d2c26fed4f7fba4a6161e143e124, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5c8a0f3ded224f5fa4bc24bc76f326dc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/221cdaeaa7e74267a09e990070d6048a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/8a0aee196b8e47259792962ca47f6e73, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/7e1e2a470c4f4b62b4dac2d28b1e69c8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/3e04e04513114811b5863649d6e0d5fe, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/218a751a963a4848b334186eaa2da8b6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/ffc1e9f5c2d3467fb6f43b67277c4339, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/f577ee5d41e8476d8a61f0870d983363, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/a409085dd1764b9986c50ba6c00455bc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/94cdecddf77e417082179ba674b0bea9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/8ac045d6058b4f9a8d3ec001fbd892b4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/fb9d3a7137f145b4894a3084015b755b] to archive 2024-12-03T09:15:23,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:15:23,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/017b709515334f99956447b9b40b8c8a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/017b709515334f99956447b9b40b8c8a 2024-12-03T09:15:23,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5cb899849ff24b8b8fa4f449f7f488b6 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5cb899849ff24b8b8fa4f449f7f488b6 2024-12-03T09:15:23,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/aac2417c16114c5eaf8b60af845529db to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/aac2417c16114c5eaf8b60af845529db 2024-12-03T09:15:23,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/c95c3fe4a6c14a9ba4f6fee1524fa5ca to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/c95c3fe4a6c14a9ba4f6fee1524fa5ca 2024-12-03T09:15:23,247 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/4ff1c63649b7446bb5e732cf478a26bd to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/4ff1c63649b7446bb5e732cf478a26bd 2024-12-03T09:15:23,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/94cc39cdba104dc5800e23ca8b8e1e98 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/94cc39cdba104dc5800e23ca8b8e1e98 2024-12-03T09:15:23,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/4b1f412421784ab59dfed23f9cad19ce to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/4b1f412421784ab59dfed23f9cad19ce 2024-12-03T09:15:23,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/7b4c5aac425a4a90aa94e1e5e34832c9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/7b4c5aac425a4a90aa94e1e5e34832c9 2024-12-03T09:15:23,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/d75a6b951bcc4b34bf0968563388df9b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/d75a6b951bcc4b34bf0968563388df9b 2024-12-03T09:15:23,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/affbbef6c2404c6faca1bf6446b92b02 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/affbbef6c2404c6faca1bf6446b92b02 2024-12-03T09:15:23,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5e08d2c26fed4f7fba4a6161e143e124 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5e08d2c26fed4f7fba4a6161e143e124 2024-12-03T09:15:23,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5c8a0f3ded224f5fa4bc24bc76f326dc to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/5c8a0f3ded224f5fa4bc24bc76f326dc 2024-12-03T09:15:23,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/221cdaeaa7e74267a09e990070d6048a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/221cdaeaa7e74267a09e990070d6048a 2024-12-03T09:15:23,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/8a0aee196b8e47259792962ca47f6e73 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/8a0aee196b8e47259792962ca47f6e73 2024-12-03T09:15:23,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/7e1e2a470c4f4b62b4dac2d28b1e69c8 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/7e1e2a470c4f4b62b4dac2d28b1e69c8 2024-12-03T09:15:23,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/3e04e04513114811b5863649d6e0d5fe to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/3e04e04513114811b5863649d6e0d5fe 2024-12-03T09:15:23,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/218a751a963a4848b334186eaa2da8b6 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/218a751a963a4848b334186eaa2da8b6 2024-12-03T09:15:23,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/ffc1e9f5c2d3467fb6f43b67277c4339 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/ffc1e9f5c2d3467fb6f43b67277c4339 2024-12-03T09:15:23,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/f577ee5d41e8476d8a61f0870d983363 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/f577ee5d41e8476d8a61f0870d983363 2024-12-03T09:15:23,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/a409085dd1764b9986c50ba6c00455bc to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/a409085dd1764b9986c50ba6c00455bc 2024-12-03T09:15:23,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/94cdecddf77e417082179ba674b0bea9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/94cdecddf77e417082179ba674b0bea9 2024-12-03T09:15:23,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/8ac045d6058b4f9a8d3ec001fbd892b4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/8ac045d6058b4f9a8d3ec001fbd892b4 2024-12-03T09:15:23,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/fb9d3a7137f145b4894a3084015b755b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/fb9d3a7137f145b4894a3084015b755b 2024-12-03T09:15:23,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/1058e7db7d214b8abc368427a3f2b3cd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/7c59d0197a1d499d99a7c882ad260251, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/1c54e983214149f9914fe4412ee5a2fb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/d15719a385f74e1a822fa998192dcb1b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/ca1de98dc89c4cf9952963469f563893, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/8bd05688007d458198644e3c3581130e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/56ee5420c833493bb6d97870d88ff0b5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/9c762d669b7b4134bcf04b09a858ed2f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/f742bf6ab7794287a710d4321e1316ef, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/b07bba194842474694c974a66ad055a5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/3013c7d8abeb43a6a02bb735325f7296, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/f3c74276f094456cbec0152f16e4cd3a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/e3d38e70e5fe42d486f3162191cf9b04, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/10404cd0adde486490b66243a7969810, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/b72661c014394ad184d957fe3e2ed935, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/29dd304410eb4576961c388def5f23f0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/7acaaa46a6b84b8e987f0d8013877de8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/20d3f72081eb41779b5c89187f6abcf1, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/9bea2e0a18f244c383574576172944cf, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/88946c3a72a4449b9076a5661cd3db4d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/08bba8a2adf94803b7b3411939ae4701, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/cbe966d11bc1461ebf8dc2bd9f469ede, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/8d1ee8b987c94e13b55325f5489e7a5d] to archive 2024-12-03T09:15:23,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:15:23,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/1058e7db7d214b8abc368427a3f2b3cd to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/1058e7db7d214b8abc368427a3f2b3cd 2024-12-03T09:15:23,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/7c59d0197a1d499d99a7c882ad260251 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/7c59d0197a1d499d99a7c882ad260251 2024-12-03T09:15:23,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/1c54e983214149f9914fe4412ee5a2fb to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/1c54e983214149f9914fe4412ee5a2fb 2024-12-03T09:15:23,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/d15719a385f74e1a822fa998192dcb1b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/d15719a385f74e1a822fa998192dcb1b 2024-12-03T09:15:23,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/ca1de98dc89c4cf9952963469f563893 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/ca1de98dc89c4cf9952963469f563893 2024-12-03T09:15:23,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/8bd05688007d458198644e3c3581130e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/8bd05688007d458198644e3c3581130e 2024-12-03T09:15:23,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/56ee5420c833493bb6d97870d88ff0b5 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/56ee5420c833493bb6d97870d88ff0b5 2024-12-03T09:15:23,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/9c762d669b7b4134bcf04b09a858ed2f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/9c762d669b7b4134bcf04b09a858ed2f 2024-12-03T09:15:23,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/f742bf6ab7794287a710d4321e1316ef to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/f742bf6ab7794287a710d4321e1316ef 2024-12-03T09:15:23,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/b07bba194842474694c974a66ad055a5 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/b07bba194842474694c974a66ad055a5 2024-12-03T09:15:23,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/3013c7d8abeb43a6a02bb735325f7296 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/3013c7d8abeb43a6a02bb735325f7296 2024-12-03T09:15:23,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/f3c74276f094456cbec0152f16e4cd3a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/f3c74276f094456cbec0152f16e4cd3a 2024-12-03T09:15:23,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/e3d38e70e5fe42d486f3162191cf9b04 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/e3d38e70e5fe42d486f3162191cf9b04 2024-12-03T09:15:23,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/10404cd0adde486490b66243a7969810 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/10404cd0adde486490b66243a7969810 2024-12-03T09:15:23,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/b72661c014394ad184d957fe3e2ed935 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/b72661c014394ad184d957fe3e2ed935 2024-12-03T09:15:23,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/29dd304410eb4576961c388def5f23f0 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/29dd304410eb4576961c388def5f23f0 2024-12-03T09:15:23,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/7acaaa46a6b84b8e987f0d8013877de8 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/7acaaa46a6b84b8e987f0d8013877de8 2024-12-03T09:15:23,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/20d3f72081eb41779b5c89187f6abcf1 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/20d3f72081eb41779b5c89187f6abcf1 2024-12-03T09:15:23,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/9bea2e0a18f244c383574576172944cf to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/9bea2e0a18f244c383574576172944cf 2024-12-03T09:15:23,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/88946c3a72a4449b9076a5661cd3db4d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/88946c3a72a4449b9076a5661cd3db4d 2024-12-03T09:15:23,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/08bba8a2adf94803b7b3411939ae4701 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/08bba8a2adf94803b7b3411939ae4701 2024-12-03T09:15:23,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/cbe966d11bc1461ebf8dc2bd9f469ede to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/cbe966d11bc1461ebf8dc2bd9f469ede 2024-12-03T09:15:23,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/8d1ee8b987c94e13b55325f5489e7a5d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/8d1ee8b987c94e13b55325f5489e7a5d 2024-12-03T09:15:23,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/447ef3e72dd443199ad31d8eca958524, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/8cb1e4ad2ce24ffeb61c85ef16bc6350, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/21b0da63b13a473da7e2b16436e0b217, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/b00c9bfacbaa4144aa3678d4511c72d7, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/c3a4e6f09f5446c48ba46adac9808757, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a996d1c899eb4eea93cc0d7a332cb08d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3c5efa916023420fa74adfad4637b47e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/2dab1da0a8024f41bca04ab69ac9b6ca, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/0e18ab576ca8449ca7a696beae784325, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/8659872b482e4d9b9836b56f1213081a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/f23227f6cfda4507a7729ae1135f6193, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/6a0db6b18851400fb0e6c4ca76aea48f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/cdb0c88c07d94d97a4b75d63d6e5e00e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3eff536e0e55419fa115cfb3a28530eb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/74e4a6e212ee431993cf4294b405253a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a1718bda873246fc8f5c1327de618790, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/6edb022f55b649f0839003dac847b710, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/f81f13cba49d41a19a16d5b7bbc82667, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3f1c9ff315d24e89a7815d816a656ece, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/ec2f5f83c725440585e3279d657b0660, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/9751512598524c21813347340df510fc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/c9e9697e48ac4939945131cc3f204e60, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a004de19181246789f6f3ff0a7164e23] to archive 2024-12-03T09:15:23,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:15:23,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/447ef3e72dd443199ad31d8eca958524 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/447ef3e72dd443199ad31d8eca958524 2024-12-03T09:15:23,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/8cb1e4ad2ce24ffeb61c85ef16bc6350 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/8cb1e4ad2ce24ffeb61c85ef16bc6350 2024-12-03T09:15:23,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/21b0da63b13a473da7e2b16436e0b217 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/21b0da63b13a473da7e2b16436e0b217 2024-12-03T09:15:23,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/b00c9bfacbaa4144aa3678d4511c72d7 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/b00c9bfacbaa4144aa3678d4511c72d7 2024-12-03T09:15:23,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/c3a4e6f09f5446c48ba46adac9808757 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/c3a4e6f09f5446c48ba46adac9808757 2024-12-03T09:15:23,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a996d1c899eb4eea93cc0d7a332cb08d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a996d1c899eb4eea93cc0d7a332cb08d 2024-12-03T09:15:23,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3c5efa916023420fa74adfad4637b47e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3c5efa916023420fa74adfad4637b47e 2024-12-03T09:15:23,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/2dab1da0a8024f41bca04ab69ac9b6ca to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/2dab1da0a8024f41bca04ab69ac9b6ca 2024-12-03T09:15:23,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/0e18ab576ca8449ca7a696beae784325 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/0e18ab576ca8449ca7a696beae784325 2024-12-03T09:15:23,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/8659872b482e4d9b9836b56f1213081a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/8659872b482e4d9b9836b56f1213081a 2024-12-03T09:15:23,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/f23227f6cfda4507a7729ae1135f6193 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/f23227f6cfda4507a7729ae1135f6193 2024-12-03T09:15:23,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/6a0db6b18851400fb0e6c4ca76aea48f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/6a0db6b18851400fb0e6c4ca76aea48f 2024-12-03T09:15:23,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/cdb0c88c07d94d97a4b75d63d6e5e00e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/cdb0c88c07d94d97a4b75d63d6e5e00e 2024-12-03T09:15:23,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3eff536e0e55419fa115cfb3a28530eb to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3eff536e0e55419fa115cfb3a28530eb 2024-12-03T09:15:23,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/74e4a6e212ee431993cf4294b405253a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/74e4a6e212ee431993cf4294b405253a 2024-12-03T09:15:23,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a1718bda873246fc8f5c1327de618790 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a1718bda873246fc8f5c1327de618790 2024-12-03T09:15:23,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/6edb022f55b649f0839003dac847b710 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/6edb022f55b649f0839003dac847b710 2024-12-03T09:15:23,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/f81f13cba49d41a19a16d5b7bbc82667 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/f81f13cba49d41a19a16d5b7bbc82667 2024-12-03T09:15:23,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3f1c9ff315d24e89a7815d816a656ece to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/3f1c9ff315d24e89a7815d816a656ece 2024-12-03T09:15:23,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/ec2f5f83c725440585e3279d657b0660 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/ec2f5f83c725440585e3279d657b0660 2024-12-03T09:15:23,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/9751512598524c21813347340df510fc to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/9751512598524c21813347340df510fc 2024-12-03T09:15:23,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/c9e9697e48ac4939945131cc3f204e60 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/c9e9697e48ac4939945131cc3f204e60 2024-12-03T09:15:23,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a004de19181246789f6f3ff0a7164e23 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/a004de19181246789f6f3ff0a7164e23 2024-12-03T09:15:23,302 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/recovered.edits/385.seqid, newMaxSeqId=385, maxSeqId=1 2024-12-03T09:15:23,303 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. 2024-12-03T09:15:23,303 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 58a142609061c77936f842676cc6c10e: Waiting for close lock at 1733217321964Running coprocessor pre-close hooks at 1733217321964Disabling compacts and flushes for region at 1733217321964Disabling writes for close at 1733217321965 (+1 ms)Obtaining lock to block concurrent updates at 1733217321965Preparing flush snapshotting stores in 58a142609061c77936f842676cc6c10e at 1733217321965Finished memstore snapshotting TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733217321966 (+1 ms)Flushing stores of TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e. at 1733217321968 (+2 ms)Flushing 58a142609061c77936f842676cc6c10e/A: creating writer at 1733217321968Flushing 58a142609061c77936f842676cc6c10e/A: appending metadata at 1733217321972 (+4 ms)Flushing 58a142609061c77936f842676cc6c10e/A: closing flushed file at 1733217321972Flushing 58a142609061c77936f842676cc6c10e/B: creating writer at 1733217322387 (+415 ms)Flushing 58a142609061c77936f842676cc6c10e/B: appending metadata at 1733217322390 (+3 ms)Flushing 58a142609061c77936f842676cc6c10e/B: closing flushed file at 1733217322390Flushing 58a142609061c77936f842676cc6c10e/C: creating writer at 1733217322804 (+414 ms)Flushing 58a142609061c77936f842676cc6c10e/C: appending metadata at 1733217322808 (+4 ms)Flushing 58a142609061c77936f842676cc6c10e/C: closing flushed file at 1733217322808Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68b5b08a: reopening flushed file at 1733217323223 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bdbcca3: reopening flushed file at 1733217323230 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6003c7d2: reopening flushed file at 1733217323236 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 58a142609061c77936f842676cc6c10e in 1276ms, sequenceid=382, compaction requested=true at 1733217323241 (+5 ms)Writing region close event to WAL at 1733217323300 (+59 ms)Running coprocessor post-close hooks at 1733217323303 (+3 ms)Closed at 1733217323303 2024-12-03T09:15:23,304 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 58a142609061c77936f842676cc6c10e 2024-12-03T09:15:23,305 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=58a142609061c77936f842676cc6c10e, regionState=CLOSED 2024-12-03T09:15:23,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:15:23,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-03T09:15:23,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 58a142609061c77936f842676cc6c10e, server=699c9baf3d29,42271,1733217224446 in 1.4960 sec 2024-12-03T09:15:23,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-03T09:15:23,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58a142609061c77936f842676cc6c10e, UNASSIGN in 1.5010 sec 2024-12-03T09:15:23,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-03T09:15:23,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5040 sec 2024-12-03T09:15:23,312 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217323312"}]},"ts":"1733217323312"} 2024-12-03T09:15:23,313 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T09:15:23,313 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T09:15:23,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5160 sec 2024-12-03T09:15:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T09:15:23,950 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:23,952 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T09:15:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:23,956 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T09:15:23,957 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:23,962 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e 2024-12-03T09:15:23,967 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/recovered.edits] 2024-12-03T09:15:24,008 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/82547a29af784130a89a1bd7e3bfa541 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/82547a29af784130a89a1bd7e3bfa541 2024-12-03T09:15:24,011 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/a1609042a98942749e6cd91e63b7223b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/a1609042a98942749e6cd91e63b7223b 2024-12-03T09:15:24,014 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/a389be1d003e4a4faddb3e79f21de33c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/A/a389be1d003e4a4faddb3e79f21de33c 2024-12-03T09:15:24,020 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/4f3c40d2ca36401c853da7cb0c74674d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/4f3c40d2ca36401c853da7cb0c74674d 2024-12-03T09:15:24,022 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/6e1cd705328c4bc1841f0743061ed080 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/6e1cd705328c4bc1841f0743061ed080 2024-12-03T09:15:24,024 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/a9834175765545a5aa89c966754a0f7a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/B/a9834175765545a5aa89c966754a0f7a 2024-12-03T09:15:24,027 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/4d2b891ac95b45f8923c063fb7a8f3bf to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/4d2b891ac95b45f8923c063fb7a8f3bf 2024-12-03T09:15:24,028 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/68d4f134cbf841a78ca0fe675b57b914 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/68d4f134cbf841a78ca0fe675b57b914 2024-12-03T09:15:24,030 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/e5cf1494f8ce41d38bdc916aaa21295c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/C/e5cf1494f8ce41d38bdc916aaa21295c 2024-12-03T09:15:24,034 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/recovered.edits/385.seqid to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e/recovered.edits/385.seqid 2024-12-03T09:15:24,034 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/58a142609061c77936f842676cc6c10e 2024-12-03T09:15:24,034 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T09:15:24,037 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:24,041 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T09:15:24,044 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T09:15:24,046 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:24,046 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T09:15:24,046 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733217324046"}]},"ts":"9223372036854775807"} 2024-12-03T09:15:24,048 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T09:15:24,048 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 58a142609061c77936f842676cc6c10e, NAME => 'TestAcidGuarantees,,1733217290639.58a142609061c77936f842676cc6c10e.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T09:15:24,048 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T09:15:24,049 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733217324048"}]},"ts":"9223372036854775807"} 2024-12-03T09:15:24,051 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T09:15:24,052 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:24,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T09:15:24,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 100 msec 2024-12-03T09:15:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T09:15:24,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T09:15:24,060 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:24,077 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=445 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 363) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12809 (was 12946) 2024-12-03T09:15:24,090 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=374, ProcessCount=11, AvailableMemoryMB=12808 2024-12-03T09:15:24,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T09:15:24,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T09:15:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:24,094 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T09:15:24,094 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:24,094 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-12-03T09:15:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T09:15:24,095 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T09:15:24,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742126_1302 (size=1061) 2024-12-03T09:15:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T09:15:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T09:15:24,506 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09 2024-12-03T09:15:24,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742127_1303 (size=53) 2024-12-03T09:15:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T09:15:24,919 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:15:24,919 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 52f05dcf6274e3c4c8ffbdf5058e1f61, disabling compactions & flushes 2024-12-03T09:15:24,919 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:24,919 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:24,919 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. after waiting 0 ms 2024-12-03T09:15:24,919 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:24,919 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:24,919 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: Waiting for close lock at 1733217324919Disabling compacts and flushes for region at 1733217324919Disabling writes for close at 1733217324919Writing region close event to WAL at 1733217324919Closed at 1733217324919 2024-12-03T09:15:24,922 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T09:15:24,922 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733217324922"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733217324922"}]},"ts":"1733217324922"} 2024-12-03T09:15:24,925 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T09:15:24,926 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T09:15:24,926 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217324926"}]},"ts":"1733217324926"} 2024-12-03T09:15:24,928 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T09:15:24,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, ASSIGN}] 2024-12-03T09:15:24,930 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, ASSIGN 2024-12-03T09:15:24,931 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, ASSIGN; state=OFFLINE, location=699c9baf3d29,42271,1733217224446; forceNewPlan=false, retain=false 2024-12-03T09:15:25,082 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=52f05dcf6274e3c4c8ffbdf5058e1f61, regionState=OPENING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:25,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, ASSIGN because future has completed 2024-12-03T09:15:25,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:15:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T09:15:25,250 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:25,250 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:15:25,251 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:25,251 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:15:25,251 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:25,251 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:25,253 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:25,255 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:15:25,255 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52f05dcf6274e3c4c8ffbdf5058e1f61 columnFamilyName A 2024-12-03T09:15:25,255 DEBUG [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:25,256 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.HStore(327): Store=52f05dcf6274e3c4c8ffbdf5058e1f61/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:15:25,256 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:25,257 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:15:25,258 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52f05dcf6274e3c4c8ffbdf5058e1f61 columnFamilyName B 2024-12-03T09:15:25,258 DEBUG [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:25,258 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.HStore(327): Store=52f05dcf6274e3c4c8ffbdf5058e1f61/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:15:25,258 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:25,260 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:15:25,260 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52f05dcf6274e3c4c8ffbdf5058e1f61 columnFamilyName C 2024-12-03T09:15:25,260 DEBUG [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:25,261 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.HStore(327): Store=52f05dcf6274e3c4c8ffbdf5058e1f61/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:15:25,261 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:25,261 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:25,262 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:25,263 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:25,264 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:25,264 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:25,265 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T09:15:25,267 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:25,269 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:15:25,269 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 52f05dcf6274e3c4c8ffbdf5058e1f61; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66764736, jitterRate=-0.005127906799316406}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T09:15:25,269 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:25,270 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: Running coprocessor pre-open hook at 1733217325251Writing region info on filesystem at 1733217325251Initializing all the Stores at 1733217325252 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217325252Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217325253 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217325253Cleaning up temporary data from old regions at 1733217325264 (+11 ms)Running coprocessor post-open hooks at 1733217325269 (+5 ms)Region opened successfully at 1733217325270 (+1 ms) 2024-12-03T09:15:25,271 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., pid=87, masterSystemTime=1733217325242 2024-12-03T09:15:25,272 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:25,272 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:25,273 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=52f05dcf6274e3c4c8ffbdf5058e1f61, regionState=OPEN, openSeqNum=2, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:25,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:15:25,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-03T09:15:25,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 in 187 msec 2024-12-03T09:15:25,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-03T09:15:25,278 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, ASSIGN in 347 msec 2024-12-03T09:15:25,279 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T09:15:25,279 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217325279"}]},"ts":"1733217325279"} 2024-12-03T09:15:25,280 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T09:15:25,281 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T09:15:25,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1890 sec 2024-12-03T09:15:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T09:15:26,242 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:26,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@200149f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:26,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:26,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:26,247 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:26,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d66eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:26,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:26,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:26,251 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:26,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23af4717, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:26,252 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:26,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:26,254 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T09:15:26,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 699c9baf3d29,35781,1733217223715 2024-12-03T09:15:26,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4db3cbfa 2024-12-03T09:15:26,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T09:15:26,256 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T09:15:26,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T09:15:26,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T09:15:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:26,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742128_1304 (size=1097) 2024-12-03T09:15:26,673 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T09:15:26,674 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T09:15:26,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T09:15:26,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, REOPEN/MOVE}] 2024-12-03T09:15:26,684 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, REOPEN/MOVE 2024-12-03T09:15:26,684 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=52f05dcf6274e3c4c8ffbdf5058e1f61, regionState=CLOSING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:26,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, REOPEN/MOVE because future has completed 2024-12-03T09:15:26,686 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T09:15:26,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:15:26,839 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:26,839 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T09:15:26,839 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 52f05dcf6274e3c4c8ffbdf5058e1f61, disabling compactions & flushes 2024-12-03T09:15:26,839 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:26,839 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:26,839 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. after waiting 0 ms 2024-12-03T09:15:26,839 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:26,847 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T09:15:26,849 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:26,849 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: Waiting for close lock at 1733217326839Running coprocessor pre-close hooks at 1733217326839Disabling compacts and flushes for region at 1733217326839Disabling writes for close at 1733217326839Writing region close event to WAL at 1733217326840 (+1 ms)Running coprocessor post-close hooks at 1733217326849 (+9 ms)Closed at 1733217326849 2024-12-03T09:15:26,849 WARN [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 52f05dcf6274e3c4c8ffbdf5058e1f61 to self. 2024-12-03T09:15:26,852 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:26,853 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=52f05dcf6274e3c4c8ffbdf5058e1f61, regionState=CLOSED 2024-12-03T09:15:26,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:15:26,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-03T09:15:26,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 in 169 msec 2024-12-03T09:15:26,858 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, REOPEN/MOVE; state=CLOSED, location=699c9baf3d29,42271,1733217224446; forceNewPlan=false, retain=true 2024-12-03T09:15:27,009 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=52f05dcf6274e3c4c8ffbdf5058e1f61, regionState=OPENING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, REOPEN/MOVE because future has completed 2024-12-03T09:15:27,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:15:27,169 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:27,169 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:15:27,170 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,170 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:15:27,170 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,170 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,172 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,173 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:15:27,174 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52f05dcf6274e3c4c8ffbdf5058e1f61 columnFamilyName A 2024-12-03T09:15:27,176 DEBUG [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:27,176 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.HStore(327): Store=52f05dcf6274e3c4c8ffbdf5058e1f61/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:15:27,177 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,178 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:15:27,178 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52f05dcf6274e3c4c8ffbdf5058e1f61 columnFamilyName B 2024-12-03T09:15:27,179 DEBUG [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:27,179 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.HStore(327): Store=52f05dcf6274e3c4c8ffbdf5058e1f61/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:15:27,180 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,180 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:15:27,180 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52f05dcf6274e3c4c8ffbdf5058e1f61 columnFamilyName C 2024-12-03T09:15:27,180 DEBUG [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:27,181 INFO [StoreOpener-52f05dcf6274e3c4c8ffbdf5058e1f61-1 {}] regionserver.HStore(327): Store=52f05dcf6274e3c4c8ffbdf5058e1f61/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:15:27,181 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:27,181 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,182 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,183 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,184 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,184 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,185 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T09:15:27,186 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,187 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 52f05dcf6274e3c4c8ffbdf5058e1f61; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67723267, jitterRate=0.009155318140983582}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T09:15:27,187 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,188 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: Running coprocessor pre-open hook at 1733217327170Writing region info on filesystem at 1733217327170Initializing all the Stores at 1733217327171 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217327171Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217327172 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217327172Cleaning up temporary data from old regions at 1733217327184 (+12 ms)Running coprocessor post-open hooks at 1733217327187 (+3 ms)Region opened successfully at 1733217327188 (+1 ms) 2024-12-03T09:15:27,189 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., pid=92, masterSystemTime=1733217327165 2024-12-03T09:15:27,190 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:27,190 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:27,191 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=52f05dcf6274e3c4c8ffbdf5058e1f61, regionState=OPEN, openSeqNum=5, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:15:27,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-12-03T09:15:27,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 in 181 msec 2024-12-03T09:15:27,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-03T09:15:27,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, REOPEN/MOVE in 512 msec 2024-12-03T09:15:27,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-03T09:15:27,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 518 msec 2024-12-03T09:15:27,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 945 msec 2024-12-03T09:15:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-03T09:15:27,205 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:27,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@773a8cf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:27,206 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:27,206 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:27,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:27,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:27,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a43688, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:27,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:27,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:27,208 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43dfa8d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a89538, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,210 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:27,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:27,210 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:27,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:27,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:27,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39459a8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:27,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:27,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:27,212 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f3d525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a9751d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,214 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:27,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:27,215 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a214cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:27,216 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:27,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@207283f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346078b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,217 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:27,218 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:27,218 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292ad4e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:27,220 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:27,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1b089b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c26e048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:27,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:27,222 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711b8a70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:27,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:27,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:27,223 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:27,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11d10f43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ce58ff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,225 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:27,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:27,226 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed1ff47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:27,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:27,227 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f26fd41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634cbd63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,229 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:27,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:27,230 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1751e615, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:27,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:27,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:27,231 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:27,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d041db1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f70d2b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,233 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:27,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:27,234 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c43e775, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:27,235 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:27,236 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@839ff46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6739e85a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,237 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:27,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:27,238 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1461a8f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,238 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:27,240 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b459471, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7de09a6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:27,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:27,242 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:27,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@694df283, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:27,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:27,244 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:27,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43a67b66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:27,245 DEBUG [Thread-1411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:27,245 DEBUG [Thread-1417 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:27,245 DEBUG [Thread-1413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:27,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:27,245 DEBUG [Thread-1424 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:27,246 DEBUG [Thread-1428 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:27,246 DEBUG [Thread-1426 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:27,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:27,247 DEBUG [Thread-1420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-03T09:15:27,247 DEBUG [Thread-1422 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:27,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:27,248 DEBUG [Thread-1415 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:27,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:27,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:27,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:27,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:27,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:27,249 DEBUG [Thread-1409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:27,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:27,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:27,250 DEBUG [Thread-1422 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:27,250 DEBUG [Thread-1422 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:27,250 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:27,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T09:15:27,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:27,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:27,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:27,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:27,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:15:27,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:27,252 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:27,252 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:27,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:15:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:15:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:15:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:15:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:15:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:15:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:15:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:15:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:15:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:15:27,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:27,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:27,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:27,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:27,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:27,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:27,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039f93709e364e4881902704a066f5d570_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217327260/Put/seqid=0 2024-12-03T09:15:27,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742129_1305 (size=14594) 2024-12-03T09:15:27,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217387312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217327310, max timestamp=1733217327318Num uniques 859; 2024-12-03T09:15:27,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217327310, max timestamp=1733217327318Num uniques 859; 2024-12-03T09:15:27,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217327310, max timestamp=1733217327318Num uniques 859; 2024-12-03T09:15:27,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:15:27,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:15:27,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:15:27,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217387348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T09:15:27,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217387353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217387354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217387355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T09:15:27,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:27,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:27,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:27,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:27,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217387429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217387460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217387469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217387469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217387469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T09:15:27,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:27,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:27,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:27,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:27,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T09:15:27,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217387640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217387669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217387679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217387679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217387680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,687 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:27,691 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039f93709e364e4881902704a066f5d570_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039f93709e364e4881902704a066f5d570_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:27,692 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/517764f5635540c1b5912a9b5647e9d9, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:27,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/517764f5635540c1b5912a9b5647e9d9 is 175, key is test_row_0/A:col10/1733217327260/Put/seqid=0 2024-12-03T09:15:27,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742130_1306 (size=39549) 2024-12-03T09:15:27,697 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/517764f5635540c1b5912a9b5647e9d9 2024-12-03T09:15:27,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T09:15:27,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:27,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:27,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:27,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:27,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:27,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/6b82c356a0f6452f9b19749520ce92e8 is 50, key is test_row_0/B:col10/1733217327260/Put/seqid=0 2024-12-03T09:15:27,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742131_1307 (size=12001) 2024-12-03T09:15:27,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/6b82c356a0f6452f9b19749520ce92e8 2024-12-03T09:15:27,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/9a1aa3c2216f4275beb459b4a3f2f82c is 50, key is test_row_0/C:col10/1733217327260/Put/seqid=0 2024-12-03T09:15:27,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742132_1308 (size=12001) 2024-12-03T09:15:27,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T09:15:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:27,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:27,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T09:15:27,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217387950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217387989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217387989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217387989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:28,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217388000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:28,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T09:15:28,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:28,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:28,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:28,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:28,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:28,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/9a1aa3c2216f4275beb459b4a3f2f82c 2024-12-03T09:15:28,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/517764f5635540c1b5912a9b5647e9d9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/517764f5635540c1b5912a9b5647e9d9 2024-12-03T09:15:28,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/517764f5635540c1b5912a9b5647e9d9, entries=200, sequenceid=15, filesize=38.6 K 2024-12-03T09:15:28,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/6b82c356a0f6452f9b19749520ce92e8 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/6b82c356a0f6452f9b19749520ce92e8 2024-12-03T09:15:28,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/6b82c356a0f6452f9b19749520ce92e8, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T09:15:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/9a1aa3c2216f4275beb459b4a3f2f82c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/9a1aa3c2216f4275beb459b4a3f2f82c 2024-12-03T09:15:28,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T09:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:28,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:28,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/9a1aa3c2216f4275beb459b4a3f2f82c, entries=150, sequenceid=15, filesize=11.7 K 2024-12-03T09:15:28,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 916ms, sequenceid=15, compaction requested=false 2024-12-03T09:15:28,176 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T09:15:28,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:28,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T09:15:28,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:28,326 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-03T09:15:28,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:28,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:28,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:28,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:28,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:28,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:28,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030a147d4ffc1e4f529910bde31fb57f2d_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217327353/Put/seqid=0 2024-12-03T09:15:28,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742133_1309 (size=12154) 2024-12-03T09:15:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T09:15:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:28,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:28,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217388499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:28,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217388500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:28,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217388509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217388509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217388509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:28,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217388609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:28,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,742 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030a147d4ffc1e4f529910bde31fb57f2d_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030a147d4ffc1e4f529910bde31fb57f2d_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:28,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/20af55685a2a4839807678dec6be7124, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:28,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/20af55685a2a4839807678dec6be7124 is 175, key is test_row_0/A:col10/1733217327353/Put/seqid=0 2024-12-03T09:15:28,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742134_1310 (size=30955) 2024-12-03T09:15:28,760 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/20af55685a2a4839807678dec6be7124 2024-12-03T09:15:28,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/32f62b6fc6f24424ae4008504617daec is 50, key is test_row_0/B:col10/1733217327353/Put/seqid=0 2024-12-03T09:15:28,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742135_1311 (size=12001) 2024-12-03T09:15:28,797 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/32f62b6fc6f24424ae4008504617daec 2024-12-03T09:15:28,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/2066dc028e904c208fc4197b4e08d23e is 50, key is test_row_0/C:col10/1733217327353/Put/seqid=0 2024-12-03T09:15:28,823 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T09:15:28,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217388819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:28,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742136_1312 (size=12001) 2024-12-03T09:15:28,890 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/2066dc028e904c208fc4197b4e08d23e 2024-12-03T09:15:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/20af55685a2a4839807678dec6be7124 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/20af55685a2a4839807678dec6be7124 2024-12-03T09:15:28,909 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/20af55685a2a4839807678dec6be7124, entries=150, sequenceid=51, filesize=30.2 K 2024-12-03T09:15:28,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/32f62b6fc6f24424ae4008504617daec as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/32f62b6fc6f24424ae4008504617daec 2024-12-03T09:15:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,919 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/32f62b6fc6f24424ae4008504617daec, entries=150, sequenceid=51, filesize=11.7 K 2024-12-03T09:15:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/2066dc028e904c208fc4197b4e08d23e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/2066dc028e904c208fc4197b4e08d23e 2024-12-03T09:15:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,928 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/2066dc028e904c208fc4197b4e08d23e, entries=150, sequenceid=51, filesize=11.7 K 2024-12-03T09:15:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,929 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 603ms, sequenceid=51, compaction requested=false 2024-12-03T09:15:28,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:28,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:28,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-03T09:15:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-03T09:15:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-03T09:15:28,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6780 sec 2024-12-03T09:15:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.6910 sec 2024-12-03T09:15:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T09:15:29,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:29,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:29,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:29,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:29,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f06051b4f10749aebfd2f3060a576029_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217328487/Put/seqid=0 2024-12-03T09:15:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742137_1313 (size=12154) 2024-12-03T09:15:29,187 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,192 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f06051b4f10749aebfd2f3060a576029_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f06051b4f10749aebfd2f3060a576029_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:29,195 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/ae94047cdb0d4e159f9bca57f51a7597, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:29,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/ae94047cdb0d4e159f9bca57f51a7597 is 175, key is test_row_0/A:col10/1733217328487/Put/seqid=0 2024-12-03T09:15:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:29,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742138_1314 (size=30955) 2024-12-03T09:15:29,211 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/ae94047cdb0d4e159f9bca57f51a7597 2024-12-03T09:15:29,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/f226258227e545fb86189bf23ac9e51f is 50, key is test_row_0/B:col10/1733217328487/Put/seqid=0 2024-12-03T09:15:29,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742139_1315 (size=12001) 2024-12-03T09:15:29,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217329293, max timestamp=1733217329321Num uniques 859; 2024-12-03T09:15:29,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217329293, max timestamp=1733217329321Num uniques 859; 2024-12-03T09:15:29,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217329293, max timestamp=1733217329321Num uniques 859; 2024-12-03T09:15:29,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:15:29,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:15:29,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:15:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T09:15:29,400 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:29,400 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-03T09:15:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T09:15:29,402 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:29,402 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:29,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:29,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217389441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T09:15:29,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217389519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:29,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217389519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:29,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217389519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:29,528 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 52f05dcf6274e3c4c8ffbdf5058e1f61 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:29,529 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 52f05dcf6274e3c4c8ffbdf5058e1f61 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:29,529 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 52f05dcf6274e3c4c8ffbdf5058e1f61 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:29,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217389530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:29,540 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 52f05dcf6274e3c4c8ffbdf5058e1f61 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:29,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T09:15:29,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:29,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:29,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:29,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:29,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:29,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217389559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:29,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:29,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/f226258227e545fb86189bf23ac9e51f 2024-12-03T09:15:29,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/741181165170440fa356f849c241460f is 50, key is test_row_0/C:col10/1733217328487/Put/seqid=0 2024-12-03T09:15:29,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742140_1316 (size=12001) 2024-12-03T09:15:29,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T09:15:29,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:29,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:29,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:29,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:29,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T09:15:29,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217389770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:29,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T09:15:29,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:29,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:29,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:29,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:29,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:30,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T09:15:30,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:30,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:30,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:30,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:30,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:30,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T09:15:30,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/741181165170440fa356f849c241460f 2024-12-03T09:15:30,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/ae94047cdb0d4e159f9bca57f51a7597 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae94047cdb0d4e159f9bca57f51a7597 2024-12-03T09:15:30,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae94047cdb0d4e159f9bca57f51a7597, entries=150, sequenceid=62, filesize=30.2 K 2024-12-03T09:15:30,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/f226258227e545fb86189bf23ac9e51f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/f226258227e545fb86189bf23ac9e51f 2024-12-03T09:15:30,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/f226258227e545fb86189bf23ac9e51f, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T09:15:30,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/741181165170440fa356f849c241460f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/741181165170440fa356f849c241460f 2024-12-03T09:15:30,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:30,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217390079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:30,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:30,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:30,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:30,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/741181165170440fa356f849c241460f, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T09:15:30,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 931ms, sequenceid=62, compaction requested=true 2024-12-03T09:15:30,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:30,086 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:30,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:30,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:30,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:30,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:30,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:30,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T09:15:30,087 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:30,087 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:30,087 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/A is initiating minor compaction (all files) 2024-12-03T09:15:30,087 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/A in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:30,087 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/517764f5635540c1b5912a9b5647e9d9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/20af55685a2a4839807678dec6be7124, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae94047cdb0d4e159f9bca57f51a7597] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=99.1 K 2024-12-03T09:15:30,087 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:30,087 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/517764f5635540c1b5912a9b5647e9d9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/20af55685a2a4839807678dec6be7124, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae94047cdb0d4e159f9bca57f51a7597] 2024-12-03T09:15:30,088 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:30,088 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 517764f5635540c1b5912a9b5647e9d9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733217327254 2024-12-03T09:15:30,088 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/B is initiating minor compaction (all files) 2024-12-03T09:15:30,088 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/B in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:30,088 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/6b82c356a0f6452f9b19749520ce92e8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/32f62b6fc6f24424ae4008504617daec, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/f226258227e545fb86189bf23ac9e51f] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=35.2 K 2024-12-03T09:15:30,088 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20af55685a2a4839807678dec6be7124, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733217327341 2024-12-03T09:15:30,088 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b82c356a0f6452f9b19749520ce92e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733217327254 2024-12-03T09:15:30,088 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae94047cdb0d4e159f9bca57f51a7597, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733217328487 2024-12-03T09:15:30,089 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 32f62b6fc6f24424ae4008504617daec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733217327341 2024-12-03T09:15:30,089 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f226258227e545fb86189bf23ac9e51f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733217328487 2024-12-03T09:15:30,095 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:30,096 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:30,097 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/c1a37515cdb94b6fb0644f8f880eb6c4 is 50, key is test_row_0/B:col10/1733217328487/Put/seqid=0 2024-12-03T09:15:30,098 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120375f21721320146dbb9bb3f96f6db464b_52f05dcf6274e3c4c8ffbdf5058e1f61 store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:30,100 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120375f21721320146dbb9bb3f96f6db464b_52f05dcf6274e3c4c8ffbdf5058e1f61, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:30,100 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120375f21721320146dbb9bb3f96f6db464b_52f05dcf6274e3c4c8ffbdf5058e1f61 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:30,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742142_1318 (size=4469) 2024-12-03T09:15:30,110 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#A#compaction#268 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:30,111 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/f83a335c97514b0989a7390e64eeede8 is 175, key is test_row_0/A:col10/1733217328487/Put/seqid=0 2024-12-03T09:15:30,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742141_1317 (size=12104) 2024-12-03T09:15:30,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742143_1319 (size=31058) 2024-12-03T09:15:30,132 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/c1a37515cdb94b6fb0644f8f880eb6c4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/c1a37515cdb94b6fb0644f8f880eb6c4 2024-12-03T09:15:30,136 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/f83a335c97514b0989a7390e64eeede8 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f83a335c97514b0989a7390e64eeede8 2024-12-03T09:15:30,136 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/B of 52f05dcf6274e3c4c8ffbdf5058e1f61 into c1a37515cdb94b6fb0644f8f880eb6c4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:30,136 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:30,136 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/B, priority=13, startTime=1733217330087; duration=0sec 2024-12-03T09:15:30,136 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:30,136 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:B 2024-12-03T09:15:30,137 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:30,139 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:30,139 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/C is initiating minor compaction (all files) 2024-12-03T09:15:30,139 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/C in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:30,139 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/9a1aa3c2216f4275beb459b4a3f2f82c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/2066dc028e904c208fc4197b4e08d23e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/741181165170440fa356f849c241460f] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=35.2 K 2024-12-03T09:15:30,139 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a1aa3c2216f4275beb459b4a3f2f82c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733217327254 2024-12-03T09:15:30,140 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2066dc028e904c208fc4197b4e08d23e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733217327341 2024-12-03T09:15:30,140 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 741181165170440fa356f849c241460f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733217328487 2024-12-03T09:15:30,142 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/A of 52f05dcf6274e3c4c8ffbdf5058e1f61 into f83a335c97514b0989a7390e64eeede8(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:30,142 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:30,142 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/A, priority=13, startTime=1733217330086; duration=0sec 2024-12-03T09:15:30,142 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:30,142 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:A 2024-12-03T09:15:30,148 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#C#compaction#269 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:30,148 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/396e486fc9534b17ae5f2fa204a5648c is 50, key is test_row_0/C:col10/1733217328487/Put/seqid=0 2024-12-03T09:15:30,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742144_1320 (size=12104) 2024-12-03T09:15:30,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T09:15:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:30,174 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T09:15:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:30,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ed0341840f4e4afb8ea1410733d0a2fb_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217329381/Put/seqid=0 2024-12-03T09:15:30,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742145_1321 (size=12154) 2024-12-03T09:15:30,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:30,196 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ed0341840f4e4afb8ea1410733d0a2fb_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ed0341840f4e4afb8ea1410733d0a2fb_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:30,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/058548458214480285c03315f7f29ea1, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:30,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/058548458214480285c03315f7f29ea1 is 175, key is test_row_0/A:col10/1733217329381/Put/seqid=0 2024-12-03T09:15:30,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742146_1322 (size=30955) 2024-12-03T09:15:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T09:15:30,562 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/396e486fc9534b17ae5f2fa204a5648c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/396e486fc9534b17ae5f2fa204a5648c 2024-12-03T09:15:30,568 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/C of 52f05dcf6274e3c4c8ffbdf5058e1f61 into 396e486fc9534b17ae5f2fa204a5648c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:30,568 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:30,568 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/C, priority=13, startTime=1733217330087; duration=0sec 2024-12-03T09:15:30,568 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:30,568 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:C 2024-12-03T09:15:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:30,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:30,606 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/058548458214480285c03315f7f29ea1 2024-12-03T09:15:30,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/9780e48c2f694bf78a183bab3975ff77 is 50, key is test_row_0/B:col10/1733217329381/Put/seqid=0 2024-12-03T09:15:30,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742147_1323 (size=12001) 2024-12-03T09:15:30,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217390642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:30,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:30,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217390759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:30,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217390970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:31,018 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/9780e48c2f694bf78a183bab3975ff77 2024-12-03T09:15:31,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/a8e316866d7d44519bafbe0687906559 is 50, key is test_row_0/C:col10/1733217329381/Put/seqid=0 2024-12-03T09:15:31,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742148_1324 (size=12001) 2024-12-03T09:15:31,040 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/a8e316866d7d44519bafbe0687906559 2024-12-03T09:15:31,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/058548458214480285c03315f7f29ea1 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/058548458214480285c03315f7f29ea1 2024-12-03T09:15:31,056 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/058548458214480285c03315f7f29ea1, entries=150, sequenceid=100, filesize=30.2 K 2024-12-03T09:15:31,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/9780e48c2f694bf78a183bab3975ff77 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/9780e48c2f694bf78a183bab3975ff77 2024-12-03T09:15:31,062 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/9780e48c2f694bf78a183bab3975ff77, entries=150, sequenceid=100, filesize=11.7 K 2024-12-03T09:15:31,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/a8e316866d7d44519bafbe0687906559 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/a8e316866d7d44519bafbe0687906559 2024-12-03T09:15:31,068 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/a8e316866d7d44519bafbe0687906559, entries=150, sequenceid=100, filesize=11.7 K 2024-12-03T09:15:31,069 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 895ms, sequenceid=100, compaction requested=false 2024-12-03T09:15:31,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:31,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:31,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-03T09:15:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-03T09:15:31,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-03T09:15:31,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6690 sec 2024-12-03T09:15:31,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.6730 sec 2024-12-03T09:15:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:31,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T09:15:31,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:31,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:31,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:31,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:31,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:31,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:31,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120371e7d3b774414d0cb685f9b6c3b417a4_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217330604/Put/seqid=0 2024-12-03T09:15:31,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742149_1325 (size=14594) 2024-12-03T09:15:31,323 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:31,329 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120371e7d3b774414d0cb685f9b6c3b417a4_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120371e7d3b774414d0cb685f9b6c3b417a4_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:31,330 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/f1210f07580d495dbfcadfff7fe1dde2, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:31,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/f1210f07580d495dbfcadfff7fe1dde2 is 175, key is test_row_0/A:col10/1733217330604/Put/seqid=0 2024-12-03T09:15:31,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742150_1326 (size=39549) 2024-12-03T09:15:31,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217391486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:31,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217391540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:31,550 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 52f05dcf6274e3c4c8ffbdf5058e1f61 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:31,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T09:15:31,552 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:31,552 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-03T09:15:31,554 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:31,555 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:31,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:31,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217391549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:31,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217391549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:31,556 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 52f05dcf6274e3c4c8ffbdf5058e1f61 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:31,562 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 52f05dcf6274e3c4c8ffbdf5058e1f61 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:31,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T09:15:31,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217391560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:31,566 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 52f05dcf6274e3c4c8ffbdf5058e1f61 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:31,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217391600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:31,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T09:15:31,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T09:15:31,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:31,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:31,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:31,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:31,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:31,757 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/f1210f07580d495dbfcadfff7fe1dde2 2024-12-03T09:15:31,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/343d766c2714427faececf63e249cf5a is 50, key is test_row_0/B:col10/1733217330604/Put/seqid=0 2024-12-03T09:15:31,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742151_1327 (size=12001) 2024-12-03T09:15:31,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/343d766c2714427faececf63e249cf5a 2024-12-03T09:15:31,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/0fd38db0f24c477785ac71dc5feedcba is 50, key is test_row_0/C:col10/1733217330604/Put/seqid=0 2024-12-03T09:15:31,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217391809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:31,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742152_1328 (size=12001) 2024-12-03T09:15:31,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/0fd38db0f24c477785ac71dc5feedcba 2024-12-03T09:15:31,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/f1210f07580d495dbfcadfff7fe1dde2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f1210f07580d495dbfcadfff7fe1dde2 2024-12-03T09:15:31,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f1210f07580d495dbfcadfff7fe1dde2, entries=200, sequenceid=113, filesize=38.6 K 2024-12-03T09:15:31,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/343d766c2714427faececf63e249cf5a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/343d766c2714427faececf63e249cf5a 2024-12-03T09:15:31,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/343d766c2714427faececf63e249cf5a, entries=150, sequenceid=113, filesize=11.7 K 2024-12-03T09:15:31,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/0fd38db0f24c477785ac71dc5feedcba as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/0fd38db0f24c477785ac71dc5feedcba 2024-12-03T09:15:31,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T09:15:31,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:31,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:31,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:31,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:31,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:31,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/0fd38db0f24c477785ac71dc5feedcba, entries=150, sequenceid=113, filesize=11.7 K 2024-12-03T09:15:31,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T09:15:31,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 597ms, sequenceid=113, compaction requested=true 2024-12-03T09:15:31,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:31,881 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:31,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:31,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:31,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:31,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:31,881 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:31,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:31,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:31,882 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:31,882 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/A is initiating minor compaction (all files) 2024-12-03T09:15:31,882 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/A in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:31,882 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f83a335c97514b0989a7390e64eeede8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/058548458214480285c03315f7f29ea1, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f1210f07580d495dbfcadfff7fe1dde2] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=99.2 K 2024-12-03T09:15:31,882 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:31,882 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f83a335c97514b0989a7390e64eeede8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/058548458214480285c03315f7f29ea1, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f1210f07580d495dbfcadfff7fe1dde2] 2024-12-03T09:15:31,882 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f83a335c97514b0989a7390e64eeede8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733217328487 2024-12-03T09:15:31,883 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 058548458214480285c03315f7f29ea1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733217329381 2024-12-03T09:15:31,883 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1210f07580d495dbfcadfff7fe1dde2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733217330604 2024-12-03T09:15:31,884 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:31,884 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/B is initiating minor compaction (all files) 2024-12-03T09:15:31,884 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/B in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:31,884 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/c1a37515cdb94b6fb0644f8f880eb6c4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/9780e48c2f694bf78a183bab3975ff77, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/343d766c2714427faececf63e249cf5a] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=35.3 K 2024-12-03T09:15:31,885 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a37515cdb94b6fb0644f8f880eb6c4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733217328487 2024-12-03T09:15:31,886 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9780e48c2f694bf78a183bab3975ff77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733217329381 2024-12-03T09:15:31,886 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 343d766c2714427faececf63e249cf5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733217330604 2024-12-03T09:15:31,903 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:31,905 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:31,906 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/d0a2d73bed2e4eb3b4571a69922a8ca3 is 50, key is test_row_0/B:col10/1733217330604/Put/seqid=0 2024-12-03T09:15:31,909 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412030d6c493089ce4be0839ef9a2e5d86866_52f05dcf6274e3c4c8ffbdf5058e1f61 store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:31,911 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412030d6c493089ce4be0839ef9a2e5d86866_52f05dcf6274e3c4c8ffbdf5058e1f61, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:31,911 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030d6c493089ce4be0839ef9a2e5d86866_52f05dcf6274e3c4c8ffbdf5058e1f61 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:31,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742154_1330 (size=4469) 2024-12-03T09:15:31,923 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#A#compaction#276 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:31,924 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/212eb10b58ab4652b68eae994b992879 is 175, key is test_row_0/A:col10/1733217330604/Put/seqid=0 2024-12-03T09:15:31,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742153_1329 (size=12207) 2024-12-03T09:15:31,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742155_1331 (size=31161) 2024-12-03T09:15:31,966 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/212eb10b58ab4652b68eae994b992879 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/212eb10b58ab4652b68eae994b992879 2024-12-03T09:15:31,971 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/A of 52f05dcf6274e3c4c8ffbdf5058e1f61 into 212eb10b58ab4652b68eae994b992879(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:31,971 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:31,971 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/A, priority=13, startTime=1733217331880; duration=0sec 2024-12-03T09:15:31,972 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:31,972 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:A 2024-12-03T09:15:31,972 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:31,973 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:31,973 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/C is initiating minor compaction (all files) 2024-12-03T09:15:31,973 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/C in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:31,973 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/396e486fc9534b17ae5f2fa204a5648c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/a8e316866d7d44519bafbe0687906559, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/0fd38db0f24c477785ac71dc5feedcba] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=35.3 K 2024-12-03T09:15:31,973 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 396e486fc9534b17ae5f2fa204a5648c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733217328487 2024-12-03T09:15:31,974 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8e316866d7d44519bafbe0687906559, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733217329381 2024-12-03T09:15:31,974 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fd38db0f24c477785ac71dc5feedcba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733217330604 2024-12-03T09:15:31,982 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:31,983 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/fa9e40cb3a084dae97256542dcc4de5a is 50, key is test_row_0/C:col10/1733217330604/Put/seqid=0 2024-12-03T09:15:32,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742156_1332 (size=12207) 2024-12-03T09:15:32,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T09:15:32,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:32,027 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T09:15:32,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:32,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:32,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:32,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:32,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:32,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:32,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034fa042d112154840b03482eaa4d1cac5_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217331484/Put/seqid=0 2024-12-03T09:15:32,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742157_1333 (size=12254) 2024-12-03T09:15:32,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T09:15:32,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217392196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:32,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:32,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:32,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:32,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:32,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217392310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:32,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:32,341 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/d0a2d73bed2e4eb3b4571a69922a8ca3 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/d0a2d73bed2e4eb3b4571a69922a8ca3 2024-12-03T09:15:32,346 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/B of 52f05dcf6274e3c4c8ffbdf5058e1f61 into d0a2d73bed2e4eb3b4571a69922a8ca3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:32,346 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:32,346 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/B, priority=13, startTime=1733217331881; duration=0sec 2024-12-03T09:15:32,346 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:32,346 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:B 2024-12-03T09:15:32,428 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/fa9e40cb3a084dae97256542dcc4de5a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/fa9e40cb3a084dae97256542dcc4de5a 2024-12-03T09:15:32,432 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/C of 52f05dcf6274e3c4c8ffbdf5058e1f61 into fa9e40cb3a084dae97256542dcc4de5a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:32,432 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:32,432 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/C, priority=13, startTime=1733217331881; duration=0sec 2024-12-03T09:15:32,432 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:32,432 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:C 2024-12-03T09:15:32,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:32,450 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034fa042d112154840b03482eaa4d1cac5_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034fa042d112154840b03482eaa4d1cac5_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:32,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/a55a728777d84e49a57de0bc63f4c691, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:32,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/a55a728777d84e49a57de0bc63f4c691 is 175, key is test_row_0/A:col10/1733217331484/Put/seqid=0 2024-12-03T09:15:32,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742158_1334 (size=31055) 2024-12-03T09:15:32,462 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/a55a728777d84e49a57de0bc63f4c691 2024-12-03T09:15:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/e65398c034aa48ad99a0f2cc0490d958 is 50, key is test_row_0/B:col10/1733217331484/Put/seqid=0 2024-12-03T09:15:32,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742159_1335 (size=12101) 2024-12-03T09:15:32,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217392530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:32,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T09:15:32,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217392839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:32,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:32,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:32,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:32,889 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/e65398c034aa48ad99a0f2cc0490d958 2024-12-03T09:15:32,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/33588709a5764ebe8e578440fa7cbdb2 is 50, key is test_row_0/C:col10/1733217331484/Put/seqid=0 2024-12-03T09:15:32,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742160_1336 (size=12101) 2024-12-03T09:15:33,325 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/33588709a5764ebe8e578440fa7cbdb2 2024-12-03T09:15:33,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/a55a728777d84e49a57de0bc63f4c691 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/a55a728777d84e49a57de0bc63f4c691 2024-12-03T09:15:33,337 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/a55a728777d84e49a57de0bc63f4c691, entries=150, sequenceid=134, filesize=30.3 K 2024-12-03T09:15:33,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/e65398c034aa48ad99a0f2cc0490d958 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/e65398c034aa48ad99a0f2cc0490d958 2024-12-03T09:15:33,342 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/e65398c034aa48ad99a0f2cc0490d958, entries=150, sequenceid=134, filesize=11.8 K 2024-12-03T09:15:33,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/33588709a5764ebe8e578440fa7cbdb2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/33588709a5764ebe8e578440fa7cbdb2 2024-12-03T09:15:33,349 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/33588709a5764ebe8e578440fa7cbdb2, entries=150, sequenceid=134, filesize=11.8 K 2024-12-03T09:15:33,350 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 1323ms, sequenceid=134, compaction requested=false 2024-12-03T09:15:33,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:33,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:33,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-03T09:15:33,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-03T09:15:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:33,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T09:15:33,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:33,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:33,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:33,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:33,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:33,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:33,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-03T09:15:33,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7970 sec 2024-12-03T09:15:33,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8050 sec 2024-12-03T09:15:33,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ad69cb471ff045deae6e1a3abbf251a2_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217332187/Put/seqid=0 2024-12-03T09:15:33,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742161_1337 (size=14794) 2024-12-03T09:15:33,386 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:33,393 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ad69cb471ff045deae6e1a3abbf251a2_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ad69cb471ff045deae6e1a3abbf251a2_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:33,393 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/e217fa0c39da49f38d8438ef13f0e524, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:33,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/e217fa0c39da49f38d8438ef13f0e524 is 175, key is test_row_0/A:col10/1733217332187/Put/seqid=0 2024-12-03T09:15:33,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742162_1338 (size=39749) 2024-12-03T09:15:33,405 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/e217fa0c39da49f38d8438ef13f0e524 2024-12-03T09:15:33,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/8fb5d6aefa254398a38ac3399b095130 is 50, key is test_row_0/B:col10/1733217332187/Put/seqid=0 2024-12-03T09:15:33,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742163_1339 (size=12151) 2024-12-03T09:15:33,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/8fb5d6aefa254398a38ac3399b095130 2024-12-03T09:15:33,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/d1f8b337da174641b1927008f0f57db7 is 50, key is test_row_0/C:col10/1733217332187/Put/seqid=0 2024-12-03T09:15:33,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742164_1340 (size=12151) 2024-12-03T09:15:33,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217393474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:33,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217393579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:33,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T09:15:33,710 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:33,710 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-03T09:15:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T09:15:33,712 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:33,712 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:33,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:33,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217393789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T09:15:33,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/d1f8b337da174641b1927008f0f57db7 2024-12-03T09:15:33,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/e217fa0c39da49f38d8438ef13f0e524 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/e217fa0c39da49f38d8438ef13f0e524 2024-12-03T09:15:33,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/e217fa0c39da49f38d8438ef13f0e524, entries=200, sequenceid=149, filesize=38.8 K 2024-12-03T09:15:33,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/8fb5d6aefa254398a38ac3399b095130 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/8fb5d6aefa254398a38ac3399b095130 2024-12-03T09:15:33,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T09:15:33,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/8fb5d6aefa254398a38ac3399b095130, entries=150, sequenceid=149, filesize=11.9 K 2024-12-03T09:15:33,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:33,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:33,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:33,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:33,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:33,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/d1f8b337da174641b1927008f0f57db7 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/d1f8b337da174641b1927008f0f57db7 2024-12-03T09:15:33,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/d1f8b337da174641b1927008f0f57db7, entries=150, sequenceid=149, filesize=11.9 K 2024-12-03T09:15:33,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 520ms, sequenceid=149, compaction requested=true 2024-12-03T09:15:33,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:33,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:33,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:33,873 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:33,873 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:33,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:33,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:33,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:33,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:33,874 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:33,874 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/B is initiating minor compaction (all files) 2024-12-03T09:15:33,874 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/B in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:33,874 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/d0a2d73bed2e4eb3b4571a69922a8ca3, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/e65398c034aa48ad99a0f2cc0490d958, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/8fb5d6aefa254398a38ac3399b095130] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=35.6 K 2024-12-03T09:15:33,874 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:33,875 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/A is initiating minor compaction (all files) 2024-12-03T09:15:33,875 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/A in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:33,875 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/212eb10b58ab4652b68eae994b992879, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/a55a728777d84e49a57de0bc63f4c691, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/e217fa0c39da49f38d8438ef13f0e524] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=99.6 K 2024-12-03T09:15:33,875 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:33,875 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/212eb10b58ab4652b68eae994b992879, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/a55a728777d84e49a57de0bc63f4c691, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/e217fa0c39da49f38d8438ef13f0e524] 2024-12-03T09:15:33,875 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a2d73bed2e4eb3b4571a69922a8ca3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733217330604 2024-12-03T09:15:33,875 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 212eb10b58ab4652b68eae994b992879, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733217330604 2024-12-03T09:15:33,876 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a55a728777d84e49a57de0bc63f4c691, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733217331432 2024-12-03T09:15:33,876 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting e65398c034aa48ad99a0f2cc0490d958, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733217331432 2024-12-03T09:15:33,876 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e217fa0c39da49f38d8438ef13f0e524, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733217332151 2024-12-03T09:15:33,876 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fb5d6aefa254398a38ac3399b095130, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733217332151 2024-12-03T09:15:33,901 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:33,913 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#B#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:33,913 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/b0e37a0e811b47ac9ace3b56a9b9460b is 50, key is test_row_0/B:col10/1733217332187/Put/seqid=0 2024-12-03T09:15:33,927 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203a600bdd24d7140298c83d2dec0727d0c_52f05dcf6274e3c4c8ffbdf5058e1f61 store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:33,929 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203a600bdd24d7140298c83d2dec0727d0c_52f05dcf6274e3c4c8ffbdf5058e1f61, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:33,929 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a600bdd24d7140298c83d2dec0727d0c_52f05dcf6274e3c4c8ffbdf5058e1f61 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:33,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742165_1341 (size=12459) 2024-12-03T09:15:33,961 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/b0e37a0e811b47ac9ace3b56a9b9460b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/b0e37a0e811b47ac9ace3b56a9b9460b 2024-12-03T09:15:33,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742166_1342 (size=4469) 2024-12-03T09:15:33,971 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/B of 52f05dcf6274e3c4c8ffbdf5058e1f61 into b0e37a0e811b47ac9ace3b56a9b9460b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:33,971 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:33,971 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/B, priority=13, startTime=1733217333873; duration=0sec 2024-12-03T09:15:33,971 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:33,971 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:B 2024-12-03T09:15:33,972 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:33,975 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#A#compaction#285 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:33,975 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/d50c7b03a19648d0909820d69baa654e is 175, key is test_row_0/A:col10/1733217332187/Put/seqid=0 2024-12-03T09:15:33,976 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:33,976 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/C is initiating minor compaction (all files) 2024-12-03T09:15:33,976 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/C in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:33,976 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/fa9e40cb3a084dae97256542dcc4de5a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/33588709a5764ebe8e578440fa7cbdb2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/d1f8b337da174641b1927008f0f57db7] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=35.6 K 2024-12-03T09:15:33,979 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting fa9e40cb3a084dae97256542dcc4de5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1733217330604 2024-12-03T09:15:33,979 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 33588709a5764ebe8e578440fa7cbdb2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733217331432 2024-12-03T09:15:33,980 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting d1f8b337da174641b1927008f0f57db7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733217332151 2024-12-03T09:15:33,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742167_1343 (size=31413) 2024-12-03T09:15:33,994 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:33,995 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/07af203a2acc4d21b19f3e90f391420c is 50, key is test_row_0/C:col10/1733217332187/Put/seqid=0 2024-12-03T09:15:34,003 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/d50c7b03a19648d0909820d69baa654e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/d50c7b03a19648d0909820d69baa654e 2024-12-03T09:15:34,013 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/A of 52f05dcf6274e3c4c8ffbdf5058e1f61 into d50c7b03a19648d0909820d69baa654e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:34,013 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:34,013 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/A, priority=13, startTime=1733217333873; duration=0sec 2024-12-03T09:15:34,013 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:34,013 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:A 2024-12-03T09:15:34,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T09:15:34,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:34,019 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T09:15:34,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:34,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:34,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:34,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:34,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:34,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T09:15:34,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742168_1344 (size=12459) 2024-12-03T09:15:34,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032cc61765b1364e008f40912783d7c92b_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217333440/Put/seqid=0 2024-12-03T09:15:34,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T09:15:34,053 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T09:15:34,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742169_1345 (size=12304) 2024-12-03T09:15:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:34,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:34,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217394190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:34,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217394309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T09:15:34,437 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/07af203a2acc4d21b19f3e90f391420c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/07af203a2acc4d21b19f3e90f391420c 2024-12-03T09:15:34,440 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/C of 52f05dcf6274e3c4c8ffbdf5058e1f61 into 07af203a2acc4d21b19f3e90f391420c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:34,440 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:34,440 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/C, priority=13, startTime=1733217333874; duration=0sec 2024-12-03T09:15:34,440 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:34,440 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:C 2024-12-03T09:15:34,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:34,473 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032cc61765b1364e008f40912783d7c92b_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032cc61765b1364e008f40912783d7c92b_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:34,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/661ff56146394616a207a89cc37242da, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:34,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/661ff56146394616a207a89cc37242da is 175, key is test_row_0/A:col10/1733217333440/Put/seqid=0 2024-12-03T09:15:34,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742170_1346 (size=31105) 2024-12-03T09:15:34,485 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/661ff56146394616a207a89cc37242da 2024-12-03T09:15:34,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/0b3064f74c9a40febe0016dfe53e20ca is 50, key is test_row_0/B:col10/1733217333440/Put/seqid=0 2024-12-03T09:15:34,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742171_1347 (size=12151) 2024-12-03T09:15:34,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:34,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217394519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:34,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:34,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:34,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:34,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217394829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:34,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T09:15:34,905 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/0b3064f74c9a40febe0016dfe53e20ca 2024-12-03T09:15:34,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/58c5f19cfc7f4c7bb8da71fb3556af7b is 50, key is test_row_0/C:col10/1733217333440/Put/seqid=0 2024-12-03T09:15:34,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742172_1348 (size=12151) 2024-12-03T09:15:35,330 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/58c5f19cfc7f4c7bb8da71fb3556af7b 2024-12-03T09:15:35,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/661ff56146394616a207a89cc37242da as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/661ff56146394616a207a89cc37242da 2024-12-03T09:15:35,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:35,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217395340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:35,346 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/661ff56146394616a207a89cc37242da, entries=150, sequenceid=170, filesize=30.4 K 2024-12-03T09:15:35,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/0b3064f74c9a40febe0016dfe53e20ca as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0b3064f74c9a40febe0016dfe53e20ca 2024-12-03T09:15:35,358 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0b3064f74c9a40febe0016dfe53e20ca, entries=150, sequenceid=170, filesize=11.9 K 2024-12-03T09:15:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/58c5f19cfc7f4c7bb8da71fb3556af7b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/58c5f19cfc7f4c7bb8da71fb3556af7b 2024-12-03T09:15:35,367 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/58c5f19cfc7f4c7bb8da71fb3556af7b, entries=150, sequenceid=170, filesize=11.9 K 2024-12-03T09:15:35,368 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 1349ms, sequenceid=170, compaction requested=false 2024-12-03T09:15:35,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:35,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:35,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-03T09:15:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-03T09:15:35,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-03T09:15:35,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6600 sec 2024-12-03T09:15:35,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.6680 sec 2024-12-03T09:15:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:35,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T09:15:35,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:35,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:35,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:35,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:35,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:35,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:35,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037cd638b186fc480d82f4f0b06fe875b1_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217334109/Put/seqid=0 2024-12-03T09:15:35,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742173_1349 (size=12304) 2024-12-03T09:15:35,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217395629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:35,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217395630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:35,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:35,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:35,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217395632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:35,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217395633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:35,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:35,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217395749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:35,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217395749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:35,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217395749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:35,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217395759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:35,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T09:15:35,860 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:35,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-03T09:15:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T09:15:35,862 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:35,862 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:35,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:35,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217395959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:35,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217395959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:35,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217395959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T09:15:35,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217395970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:35,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T09:15:36,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:36,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:36,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:36,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:36,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:36,021 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:36,024 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037cd638b186fc480d82f4f0b06fe875b1_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037cd638b186fc480d82f4f0b06fe875b1_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:36,025 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/ab410b23bda245feabddcff5cbf5847e, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:36,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/ab410b23bda245feabddcff5cbf5847e is 175, key is test_row_0/A:col10/1733217334109/Put/seqid=0 2024-12-03T09:15:36,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742174_1350 (size=31105) 2024-12-03T09:15:36,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T09:15:36,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:36,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:36,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:36,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:36,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T09:15:36,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217396269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217396280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:36,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217396280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217396280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T09:15:36,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:36,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:36,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:36,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:36,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:36,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217396359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:36,365 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 52f05dcf6274e3c4c8ffbdf5058e1f61 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,434 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/ab410b23bda245feabddcff5cbf5847e 2024-12-03T09:15:36,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/599a6b3a1eb640ebbaa32b7c5d67e888 is 50, key is test_row_0/B:col10/1733217334109/Put/seqid=0 2024-12-03T09:15:36,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742175_1351 (size=12151) 2024-12-03T09:15:36,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/599a6b3a1eb640ebbaa32b7c5d67e888 2024-12-03T09:15:36,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/04d64c9f51394e8c9513e72d0afa7549 is 50, key is test_row_0/C:col10/1733217334109/Put/seqid=0 2024-12-03T09:15:36,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742176_1352 (size=12151) 2024-12-03T09:15:36,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/04d64c9f51394e8c9513e72d0afa7549 2024-12-03T09:15:36,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/ab410b23bda245feabddcff5cbf5847e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ab410b23bda245feabddcff5cbf5847e 2024-12-03T09:15:36,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T09:15:36,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:36,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:36,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:36,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:36,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:36,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ab410b23bda245feabddcff5cbf5847e, entries=150, sequenceid=185, filesize=30.4 K 2024-12-03T09:15:36,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/599a6b3a1eb640ebbaa32b7c5d67e888 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/599a6b3a1eb640ebbaa32b7c5d67e888 2024-12-03T09:15:36,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/599a6b3a1eb640ebbaa32b7c5d67e888, entries=150, sequenceid=185, filesize=11.9 K 2024-12-03T09:15:36,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/04d64c9f51394e8c9513e72d0afa7549 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/04d64c9f51394e8c9513e72d0afa7549 2024-12-03T09:15:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T09:15:36,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/04d64c9f51394e8c9513e72d0afa7549, entries=150, sequenceid=185, filesize=11.9 K 2024-12-03T09:15:36,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 908ms, sequenceid=185, compaction requested=true 2024-12-03T09:15:36,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:36,490 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:36,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:36,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:36,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:36,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:36,491 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:36,491 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:36,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:36,491 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/A is initiating minor compaction (all files) 2024-12-03T09:15:36,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:36,491 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/A in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:36,492 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/d50c7b03a19648d0909820d69baa654e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/661ff56146394616a207a89cc37242da, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ab410b23bda245feabddcff5cbf5847e] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=91.4 K 2024-12-03T09:15:36,492 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:36,492 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/d50c7b03a19648d0909820d69baa654e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/661ff56146394616a207a89cc37242da, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ab410b23bda245feabddcff5cbf5847e] 2024-12-03T09:15:36,492 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:36,492 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/B is initiating minor compaction (all files) 2024-12-03T09:15:36,492 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/B in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:36,492 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d50c7b03a19648d0909820d69baa654e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733217332151 2024-12-03T09:15:36,492 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/b0e37a0e811b47ac9ace3b56a9b9460b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0b3064f74c9a40febe0016dfe53e20ca, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/599a6b3a1eb640ebbaa32b7c5d67e888] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=35.9 K 2024-12-03T09:15:36,492 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 661ff56146394616a207a89cc37242da, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733217333440 2024-12-03T09:15:36,492 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting b0e37a0e811b47ac9ace3b56a9b9460b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733217332151 2024-12-03T09:15:36,493 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab410b23bda245feabddcff5cbf5847e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733217334109 2024-12-03T09:15:36,493 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b3064f74c9a40febe0016dfe53e20ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733217333440 2024-12-03T09:15:36,493 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 599a6b3a1eb640ebbaa32b7c5d67e888, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733217334109 2024-12-03T09:15:36,509 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:36,511 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#B#compaction#295 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:36,512 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/0ee47571021d4d719f17364f8f056c93 is 50, key is test_row_0/B:col10/1733217334109/Put/seqid=0 2024-12-03T09:15:36,512 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120340a91559706f4c2a945f06c007f6c046_52f05dcf6274e3c4c8ffbdf5058e1f61 store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:36,514 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120340a91559706f4c2a945f06c007f6c046_52f05dcf6274e3c4c8ffbdf5058e1f61, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:36,514 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120340a91559706f4c2a945f06c007f6c046_52f05dcf6274e3c4c8ffbdf5058e1f61 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:36,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742177_1353 (size=12561) 2024-12-03T09:15:36,531 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/0ee47571021d4d719f17364f8f056c93 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0ee47571021d4d719f17364f8f056c93 2024-12-03T09:15:36,538 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/B of 52f05dcf6274e3c4c8ffbdf5058e1f61 into 0ee47571021d4d719f17364f8f056c93(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:36,538 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:36,538 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/B, priority=13, startTime=1733217336491; duration=0sec 2024-12-03T09:15:36,538 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:36,538 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:B 2024-12-03T09:15:36,538 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:36,539 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:36,539 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/C is initiating minor compaction (all files) 2024-12-03T09:15:36,539 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/C in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:36,539 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/07af203a2acc4d21b19f3e90f391420c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/58c5f19cfc7f4c7bb8da71fb3556af7b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/04d64c9f51394e8c9513e72d0afa7549] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=35.9 K 2024-12-03T09:15:36,539 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 07af203a2acc4d21b19f3e90f391420c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733217332151 2024-12-03T09:15:36,540 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 58c5f19cfc7f4c7bb8da71fb3556af7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733217333440 2024-12-03T09:15:36,541 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 04d64c9f51394e8c9513e72d0afa7549, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733217334109 2024-12-03T09:15:36,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742178_1354 (size=4469) 2024-12-03T09:15:36,547 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:36,548 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/a4a5d65e07cc4c9292f7c77fd737d6d2 is 50, key is test_row_0/C:col10/1733217334109/Put/seqid=0 2024-12-03T09:15:36,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742179_1355 (size=12561) 2024-12-03T09:15:36,553 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#A#compaction#294 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:36,554 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/6a3f8223f366410088e313e05d9a33cd is 175, key is test_row_0/A:col10/1733217334109/Put/seqid=0 2024-12-03T09:15:36,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742180_1356 (size=31515) 2024-12-03T09:15:36,571 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/a4a5d65e07cc4c9292f7c77fd737d6d2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/a4a5d65e07cc4c9292f7c77fd737d6d2 2024-12-03T09:15:36,576 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/C of 52f05dcf6274e3c4c8ffbdf5058e1f61 into a4a5d65e07cc4c9292f7c77fd737d6d2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:36,577 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:36,577 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/C, priority=13, startTime=1733217336491; duration=0sec 2024-12-03T09:15:36,577 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:36,577 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:C 2024-12-03T09:15:36,578 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/6a3f8223f366410088e313e05d9a33cd as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/6a3f8223f366410088e313e05d9a33cd 2024-12-03T09:15:36,592 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/A of 52f05dcf6274e3c4c8ffbdf5058e1f61 into 6a3f8223f366410088e313e05d9a33cd(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:36,592 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:36,592 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/A, priority=13, startTime=1733217336490; duration=0sec 2024-12-03T09:15:36,592 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:36,592 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:A 2024-12-03T09:15:36,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T09:15:36,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:36,630 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T09:15:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:36,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d64f7b8088d74ccab62d7582cc758c2f_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217335629/Put/seqid=0 2024-12-03T09:15:36,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742181_1357 (size=12304) 2024-12-03T09:15:36,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:36,691 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d64f7b8088d74ccab62d7582cc758c2f_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d64f7b8088d74ccab62d7582cc758c2f_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:36,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/39cb7444bc2a402c90d76a387e3596f4, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:36,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/39cb7444bc2a402c90d76a387e3596f4 is 175, key is test_row_0/A:col10/1733217335629/Put/seqid=0 2024-12-03T09:15:36,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742182_1358 (size=31105) 2024-12-03T09:15:36,719 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/39cb7444bc2a402c90d76a387e3596f4 2024-12-03T09:15:36,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/1d3a1c6b670b4a13b2732cc269557507 is 50, key is test_row_0/B:col10/1733217335629/Put/seqid=0 2024-12-03T09:15:36,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742183_1359 (size=12151) 2024-12-03T09:15:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:36,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:36,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217396821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:36,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217396827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217396828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:36,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217396831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217396940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:36,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217396940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217396940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217396940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:36,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T09:15:37,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217397150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:37,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217397150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217397150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:37,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:37,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217397150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:37,159 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/1d3a1c6b670b4a13b2732cc269557507 2024-12-03T09:15:37,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/160bd8451a4948009a88ff539686bd52 is 50, key is test_row_0/C:col10/1733217335629/Put/seqid=0 2024-12-03T09:15:37,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742184_1360 (size=12151) 2024-12-03T09:15:37,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217397460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:37,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217397460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217397460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217397460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:37,569 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/160bd8451a4948009a88ff539686bd52 2024-12-03T09:15:37,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/39cb7444bc2a402c90d76a387e3596f4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/39cb7444bc2a402c90d76a387e3596f4 2024-12-03T09:15:37,581 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/39cb7444bc2a402c90d76a387e3596f4, entries=150, sequenceid=206, filesize=30.4 K 2024-12-03T09:15:37,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/1d3a1c6b670b4a13b2732cc269557507 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/1d3a1c6b670b4a13b2732cc269557507 2024-12-03T09:15:37,590 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/1d3a1c6b670b4a13b2732cc269557507, entries=150, sequenceid=206, filesize=11.9 K 2024-12-03T09:15:37,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/160bd8451a4948009a88ff539686bd52 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/160bd8451a4948009a88ff539686bd52 2024-12-03T09:15:37,596 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/160bd8451a4948009a88ff539686bd52, entries=150, sequenceid=206, filesize=11.9 K 2024-12-03T09:15:37,597 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 967ms, sequenceid=206, compaction requested=false 2024-12-03T09:15:37,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:37,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:37,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-03T09:15:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-03T09:15:37,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-03T09:15:37,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7360 sec 2024-12-03T09:15:37,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.7430 sec 2024-12-03T09:15:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:37,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T09:15:37,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:37,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:37,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:37,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:37,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:37,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:37,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f42551ffc8d14a5a9af677c96b0762f6_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217337973/Put/seqid=0 2024-12-03T09:15:37,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742185_1361 (size=14794) 2024-12-03T09:15:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T09:15:38,010 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:38,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-03T09:15:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T09:15:38,012 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:38,013 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:38,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:38,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217398004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217398006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217398007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217398008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T09:15:38,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217398120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217398129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217398129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217398129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T09:15:38,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:38,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:38,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:38,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T09:15:38,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:38,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:38,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:38,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T09:15:38,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217398339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217398339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217398349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217398350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,389 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:38,392 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f42551ffc8d14a5a9af677c96b0762f6_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f42551ffc8d14a5a9af677c96b0762f6_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:38,392 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/d117de5cdd0c49778311778f17dc184f, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:38,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/d117de5cdd0c49778311778f17dc184f is 175, key is test_row_0/A:col10/1733217337973/Put/seqid=0 2024-12-03T09:15:38,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217398389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,394 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 52f05dcf6274e3c4c8ffbdf5058e1f61 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742186_1362 (size=39749) 2024-12-03T09:15:38,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T09:15:38,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:38,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:38,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:38,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T09:15:38,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:38,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:38,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:38,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T09:15:38,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217398649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217398649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217398659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217398660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:38,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T09:15:38,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:38,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:38,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:38,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,797 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/d117de5cdd0c49778311778f17dc184f 2024-12-03T09:15:38,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/724c97c28af14d8b9a215e0ea04fb9c3 is 50, key is test_row_0/B:col10/1733217337973/Put/seqid=0 2024-12-03T09:15:38,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742187_1363 (size=12151) 2024-12-03T09:15:38,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T09:15:38,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:38,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:38,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:38,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:39,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T09:15:39,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:39,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:39,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:39,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:39,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T09:15:39,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217399159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:39,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217399159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:39,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217399170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:39,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217399170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:39,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:39,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/724c97c28af14d8b9a215e0ea04fb9c3 2024-12-03T09:15:39,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/da77fd70046f4c3d84b9e61c0905aec8 is 50, key is test_row_0/C:col10/1733217337973/Put/seqid=0 2024-12-03T09:15:39,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742188_1364 (size=12151) 2024-12-03T09:15:39,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T09:15:39,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:39,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:39,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:39,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:39,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:39,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T09:15:39,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:39,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:39,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:39,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:39,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:39,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T09:15:39,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:39,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:39,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:39,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:39,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:39,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/da77fd70046f4c3d84b9e61c0905aec8 2024-12-03T09:15:39,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/d117de5cdd0c49778311778f17dc184f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/d117de5cdd0c49778311778f17dc184f 2024-12-03T09:15:39,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/d117de5cdd0c49778311778f17dc184f, entries=200, sequenceid=221, filesize=38.8 K 2024-12-03T09:15:39,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/724c97c28af14d8b9a215e0ea04fb9c3 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/724c97c28af14d8b9a215e0ea04fb9c3 2024-12-03T09:15:39,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/724c97c28af14d8b9a215e0ea04fb9c3, entries=150, sequenceid=221, filesize=11.9 K 2024-12-03T09:15:39,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/da77fd70046f4c3d84b9e61c0905aec8 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/da77fd70046f4c3d84b9e61c0905aec8 2024-12-03T09:15:39,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/da77fd70046f4c3d84b9e61c0905aec8, entries=150, sequenceid=221, filesize=11.9 K 2024-12-03T09:15:39,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 1674ms, sequenceid=221, compaction requested=true 2024-12-03T09:15:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:39,648 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:39,648 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:39,649 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:39,649 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/A is initiating minor compaction (all files) 2024-12-03T09:15:39,649 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/A in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:39,649 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/6a3f8223f366410088e313e05d9a33cd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/39cb7444bc2a402c90d76a387e3596f4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/d117de5cdd0c49778311778f17dc184f] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=100.0 K 2024-12-03T09:15:39,649 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:39,649 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/6a3f8223f366410088e313e05d9a33cd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/39cb7444bc2a402c90d76a387e3596f4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/d117de5cdd0c49778311778f17dc184f] 2024-12-03T09:15:39,650 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a3f8223f366410088e313e05d9a33cd, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733217334109 2024-12-03T09:15:39,650 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39cb7444bc2a402c90d76a387e3596f4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733217335624 2024-12-03T09:15:39,650 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d117de5cdd0c49778311778f17dc184f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733217336819 2024-12-03T09:15:39,654 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:39,654 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/B is initiating minor compaction (all files) 2024-12-03T09:15:39,654 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/B in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:39,654 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0ee47571021d4d719f17364f8f056c93, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/1d3a1c6b670b4a13b2732cc269557507, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/724c97c28af14d8b9a215e0ea04fb9c3] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=36.0 K 2024-12-03T09:15:39,655 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ee47571021d4d719f17364f8f056c93, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733217334109 2024-12-03T09:15:39,655 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d3a1c6b670b4a13b2732cc269557507, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733217335624 2024-12-03T09:15:39,659 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 724c97c28af14d8b9a215e0ea04fb9c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733217336819 2024-12-03T09:15:39,663 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:39,664 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203a77bcca6e72649a3ac400c1b355370b3_52f05dcf6274e3c4c8ffbdf5058e1f61 store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:39,666 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#B#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:39,666 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203a77bcca6e72649a3ac400c1b355370b3_52f05dcf6274e3c4c8ffbdf5058e1f61, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:39,667 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a77bcca6e72649a3ac400c1b355370b3_52f05dcf6274e3c4c8ffbdf5058e1f61 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:39,668 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/31beb7cf90434162a0581c65c0e51a1b is 50, key is test_row_0/B:col10/1733217337973/Put/seqid=0 2024-12-03T09:15:39,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742189_1365 (size=4469) 2024-12-03T09:15:39,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742190_1366 (size=12663) 2024-12-03T09:15:39,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T09:15:39,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:39,700 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T09:15:39,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:39,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:39,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:39,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:39,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:39,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:39,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033a7d7c83d8f847e3858573ee682849f5_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217338007/Put/seqid=0 2024-12-03T09:15:39,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742191_1367 (size=12304) 2024-12-03T09:15:40,091 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#A#compaction#303 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:40,091 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/22ceb9d44f8e4db0a3cb927dca66b3e5 is 175, key is test_row_0/A:col10/1733217337973/Put/seqid=0 2024-12-03T09:15:40,094 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/31beb7cf90434162a0581c65c0e51a1b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/31beb7cf90434162a0581c65c0e51a1b 2024-12-03T09:15:40,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742192_1368 (size=31617) 2024-12-03T09:15:40,107 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/22ceb9d44f8e4db0a3cb927dca66b3e5 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/22ceb9d44f8e4db0a3cb927dca66b3e5 2024-12-03T09:15:40,107 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/B of 52f05dcf6274e3c4c8ffbdf5058e1f61 into 31beb7cf90434162a0581c65c0e51a1b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:40,107 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:40,107 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/B, priority=13, startTime=1733217339648; duration=0sec 2024-12-03T09:15:40,107 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:40,107 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:B 2024-12-03T09:15:40,107 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:40,108 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:40,108 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/C is initiating minor compaction (all files) 2024-12-03T09:15:40,108 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/C in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:40,108 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/a4a5d65e07cc4c9292f7c77fd737d6d2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/160bd8451a4948009a88ff539686bd52, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/da77fd70046f4c3d84b9e61c0905aec8] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=36.0 K 2024-12-03T09:15:40,109 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting a4a5d65e07cc4c9292f7c77fd737d6d2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733217334109 2024-12-03T09:15:40,109 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 160bd8451a4948009a88ff539686bd52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733217335624 2024-12-03T09:15:40,109 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting da77fd70046f4c3d84b9e61c0905aec8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733217336819 2024-12-03T09:15:40,113 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/A of 52f05dcf6274e3c4c8ffbdf5058e1f61 into 22ceb9d44f8e4db0a3cb927dca66b3e5(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:40,113 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:40,113 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/A, priority=13, startTime=1733217339648; duration=0sec 2024-12-03T09:15:40,113 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:40,114 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:A 2024-12-03T09:15:40,117 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#C#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:40,117 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/e3f367752a1944e382e37b7de0f9d49e is 50, key is test_row_0/C:col10/1733217337973/Put/seqid=0 2024-12-03T09:15:40,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:40,128 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033a7d7c83d8f847e3858573ee682849f5_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033a7d7c83d8f847e3858573ee682849f5_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:40,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/ae0044b8c9ec482b841dec305da75cb0, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:40,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/ae0044b8c9ec482b841dec305da75cb0 is 175, key is test_row_0/A:col10/1733217338007/Put/seqid=0 2024-12-03T09:15:40,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742193_1369 (size=12663) 2024-12-03T09:15:40,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742194_1370 (size=31105) 2024-12-03T09:15:40,139 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/e3f367752a1944e382e37b7de0f9d49e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/e3f367752a1944e382e37b7de0f9d49e 2024-12-03T09:15:40,143 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/C of 52f05dcf6274e3c4c8ffbdf5058e1f61 into e3f367752a1944e382e37b7de0f9d49e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:40,143 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:40,143 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/C, priority=13, startTime=1733217339648; duration=0sec 2024-12-03T09:15:40,143 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:40,143 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:C 2024-12-03T09:15:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T09:15:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:40,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:40,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217400199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217400224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217400224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217400224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217400330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217400339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217400339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217400339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,535 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/ae0044b8c9ec482b841dec305da75cb0 2024-12-03T09:15:40,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217400540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/4b63ac71ac484b76ac531ddf7d4e648d is 50, key is test_row_0/B:col10/1733217338007/Put/seqid=0 2024-12-03T09:15:40,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217400549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217400550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217400550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742195_1371 (size=12151) 2024-12-03T09:15:40,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217400849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217400859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217400859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217400860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:40,967 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/4b63ac71ac484b76ac531ddf7d4e648d 2024-12-03T09:15:40,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/53aeb059598a48708400e36cef72fb98 is 50, key is test_row_0/C:col10/1733217338007/Put/seqid=0 2024-12-03T09:15:40,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742196_1372 (size=12151) 2024-12-03T09:15:41,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217401359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:41,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:41,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:41,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:41,381 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/53aeb059598a48708400e36cef72fb98 2024-12-03T09:15:41,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217401380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:41,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217401380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:41,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:41,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:41,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217401380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:41,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:41,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/ae0044b8c9ec482b841dec305da75cb0 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae0044b8c9ec482b841dec305da75cb0 2024-12-03T09:15:41,393 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae0044b8c9ec482b841dec305da75cb0, entries=150, sequenceid=238, filesize=30.4 K 2024-12-03T09:15:41,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/4b63ac71ac484b76ac531ddf7d4e648d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/4b63ac71ac484b76ac531ddf7d4e648d 2024-12-03T09:15:41,398 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/4b63ac71ac484b76ac531ddf7d4e648d, entries=150, sequenceid=238, filesize=11.9 K 2024-12-03T09:15:41,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/53aeb059598a48708400e36cef72fb98 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/53aeb059598a48708400e36cef72fb98 2024-12-03T09:15:41,402 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/53aeb059598a48708400e36cef72fb98, entries=150, sequenceid=238, filesize=11.9 K 2024-12-03T09:15:41,403 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 1703ms, sequenceid=238, compaction requested=false 2024-12-03T09:15:41,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:41,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:41,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-03T09:15:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-03T09:15:41,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-03T09:15:41,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3910 sec 2024-12-03T09:15:41,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.3970 sec 2024-12-03T09:15:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T09:15:42,179 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:42,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-03T09:15:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T09:15:42,181 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:42,182 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:42,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:42,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T09:15:42,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-03T09:15:42,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:42,334 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T09:15:42,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:42,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:42,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:42,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:42,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:42,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:42,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f9b8eacf753046ceb30d7b216793baad_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217340190/Put/seqid=0 2024-12-03T09:15:42,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742197_1373 (size=12354) 2024-12-03T09:15:42,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:42,353 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f9b8eacf753046ceb30d7b216793baad_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f9b8eacf753046ceb30d7b216793baad_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/42b4b5f682094d469a8d16e22f51fbfe, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/42b4b5f682094d469a8d16e22f51fbfe is 175, key is test_row_0/A:col10/1733217340190/Put/seqid=0 2024-12-03T09:15:42,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742198_1374 (size=31155) 2024-12-03T09:15:42,366 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/42b4b5f682094d469a8d16e22f51fbfe 2024-12-03T09:15:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:42,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/4c7d83f2ba0742d48bda6e021c072909 is 50, key is test_row_0/B:col10/1733217340190/Put/seqid=0 2024-12-03T09:15:42,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742199_1375 (size=12201) 2024-12-03T09:15:42,385 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/4c7d83f2ba0742d48bda6e021c072909 2024-12-03T09:15:42,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/d5ff4fee56014d0c95498624d530028d is 50, key is test_row_0/C:col10/1733217340190/Put/seqid=0 2024-12-03T09:15:42,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742200_1376 (size=12201) 2024-12-03T09:15:42,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217402406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:42,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217402413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:42,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217402413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:42,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:42,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217402414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:42,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46122 deadline: 1733217402430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:42,440 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 52f05dcf6274e3c4c8ffbdf5058e1f61 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T09:15:42,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217402519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:42,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:42,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217402529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:42,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217402529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:42,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217402529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:42,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217402729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:42,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217402740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217402740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217402740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:42,799 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T09:15:42,800 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/d5ff4fee56014d0c95498624d530028d 2024-12-03T09:15:42,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/42b4b5f682094d469a8d16e22f51fbfe as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/42b4b5f682094d469a8d16e22f51fbfe 2024-12-03T09:15:42,806 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/42b4b5f682094d469a8d16e22f51fbfe, entries=150, sequenceid=258, filesize=30.4 K 2024-12-03T09:15:42,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/4c7d83f2ba0742d48bda6e021c072909 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/4c7d83f2ba0742d48bda6e021c072909 2024-12-03T09:15:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T09:15:42,810 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/4c7d83f2ba0742d48bda6e021c072909, entries=150, sequenceid=258, filesize=11.9 K 2024-12-03T09:15:42,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/d5ff4fee56014d0c95498624d530028d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/d5ff4fee56014d0c95498624d530028d 2024-12-03T09:15:42,814 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/d5ff4fee56014d0c95498624d530028d, entries=150, sequenceid=258, filesize=11.9 K 2024-12-03T09:15:42,815 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 481ms, sequenceid=258, compaction requested=true 2024-12-03T09:15:42,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:42,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:42,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-03T09:15:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-03T09:15:42,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-03T09:15:42,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 634 msec 2024-12-03T09:15:42,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 638 msec 2024-12-03T09:15:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:43,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T09:15:43,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:43,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:43,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:43,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:43,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:43,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:43,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a1bbb2da771b4e93aeb21d90d69dfbab_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217342412/Put/seqid=0 2024-12-03T09:15:43,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742201_1377 (size=12454) 2024-12-03T09:15:43,071 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:43,077 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a1bbb2da771b4e93aeb21d90d69dfbab_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a1bbb2da771b4e93aeb21d90d69dfbab_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:43,078 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/12ba0070c80e4e51bc73541c4daab0a4, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:43,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/12ba0070c80e4e51bc73541c4daab0a4 is 175, key is test_row_0/A:col10/1733217342412/Put/seqid=0 2024-12-03T09:15:43,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217403076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217403081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742202_1378 (size=31255) 2024-12-03T09:15:43,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217403085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217403086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,100 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/12ba0070c80e4e51bc73541c4daab0a4 2024-12-03T09:15:43,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/b5f49e436c4c4095878dcf308505961e is 50, key is test_row_0/B:col10/1733217342412/Put/seqid=0 2024-12-03T09:15:43,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742203_1379 (size=12301) 2024-12-03T09:15:43,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217403189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217403199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217403199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217403200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T09:15:43,320 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:43,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-03T09:15:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T09:15:43,322 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:43,322 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:43,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:43,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217403399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217403410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217403410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217403410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T09:15:43,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T09:15:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:43,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:43,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:43,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:43,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:43,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:43,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/b5f49e436c4c4095878dcf308505961e 2024-12-03T09:15:43,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/1ff3ca6417184aefa06ce8bbcd5c47ab is 50, key is test_row_0/C:col10/1733217342412/Put/seqid=0 2024-12-03T09:15:43,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742204_1380 (size=12301) 2024-12-03T09:15:43,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T09:15:43,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:43,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:43,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:43,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:43,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T09:15:43,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217403709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217403719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217403720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217403720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:43,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T09:15:43,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:43,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:43,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:43,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:43,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:43,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/1ff3ca6417184aefa06ce8bbcd5c47ab 2024-12-03T09:15:43,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/12ba0070c80e4e51bc73541c4daab0a4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/12ba0070c80e4e51bc73541c4daab0a4 2024-12-03T09:15:43,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T09:15:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:43,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:43,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/12ba0070c80e4e51bc73541c4daab0a4, entries=150, sequenceid=274, filesize=30.5 K 2024-12-03T09:15:43,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/b5f49e436c4c4095878dcf308505961e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/b5f49e436c4c4095878dcf308505961e 2024-12-03T09:15:43,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/b5f49e436c4c4095878dcf308505961e, entries=150, sequenceid=274, filesize=12.0 K 2024-12-03T09:15:43,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/1ff3ca6417184aefa06ce8bbcd5c47ab as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/1ff3ca6417184aefa06ce8bbcd5c47ab 2024-12-03T09:15:43,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/1ff3ca6417184aefa06ce8bbcd5c47ab, entries=150, sequenceid=274, filesize=12.0 K 2024-12-03T09:15:43,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 899ms, sequenceid=274, compaction requested=true 2024-12-03T09:15:43,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:43,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:43,945 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:15:43,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:43,946 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:15:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:43,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:43,946 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49316 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:15:43,946 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125132 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:15:43,946 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/A is initiating minor compaction (all files) 2024-12-03T09:15:43,946 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/B is initiating minor compaction (all files) 2024-12-03T09:15:43,947 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/B in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:43,947 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/A in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:43,947 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/22ceb9d44f8e4db0a3cb927dca66b3e5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae0044b8c9ec482b841dec305da75cb0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/42b4b5f682094d469a8d16e22f51fbfe, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/12ba0070c80e4e51bc73541c4daab0a4] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=122.2 K 2024-12-03T09:15:43,947 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/31beb7cf90434162a0581c65c0e51a1b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/4b63ac71ac484b76ac531ddf7d4e648d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/4c7d83f2ba0742d48bda6e021c072909, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/b5f49e436c4c4095878dcf308505961e] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=48.2 K 2024-12-03T09:15:43,947 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:43,947 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/22ceb9d44f8e4db0a3cb927dca66b3e5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae0044b8c9ec482b841dec305da75cb0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/42b4b5f682094d469a8d16e22f51fbfe, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/12ba0070c80e4e51bc73541c4daab0a4] 2024-12-03T09:15:43,947 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 31beb7cf90434162a0581c65c0e51a1b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733217336819 2024-12-03T09:15:43,947 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22ceb9d44f8e4db0a3cb927dca66b3e5, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733217336819 2024-12-03T09:15:43,948 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b63ac71ac484b76ac531ddf7d4e648d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733217337992 2024-12-03T09:15:43,948 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae0044b8c9ec482b841dec305da75cb0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733217337992 2024-12-03T09:15:43,948 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c7d83f2ba0742d48bda6e021c072909, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733217340190 2024-12-03T09:15:43,948 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b4b5f682094d469a8d16e22f51fbfe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733217340190 2024-12-03T09:15:43,948 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting b5f49e436c4c4095878dcf308505961e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733217342411 2024-12-03T09:15:43,948 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12ba0070c80e4e51bc73541c4daab0a4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733217342411 2024-12-03T09:15:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T09:15:43,957 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:43,957 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#B#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:43,957 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/a02c6dc8c7ae4886886e10a841c07857 is 50, key is test_row_0/B:col10/1733217342412/Put/seqid=0 2024-12-03T09:15:43,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742205_1381 (size=12949) 2024-12-03T09:15:43,971 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203ec11103a68bc4038a28e23c42c6e7351_52f05dcf6274e3c4c8ffbdf5058e1f61 store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:43,972 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203ec11103a68bc4038a28e23c42c6e7351_52f05dcf6274e3c4c8ffbdf5058e1f61, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:43,972 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ec11103a68bc4038a28e23c42c6e7351_52f05dcf6274e3c4c8ffbdf5058e1f61 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:43,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742206_1382 (size=4469) 2024-12-03T09:15:44,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-03T09:15:44,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:44,085 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T09:15:44,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:44,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:44,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:44,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:44,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:44,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:44,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ff0c3ab8d8d84c5dafea0fa1b59a2c3c_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217343085/Put/seqid=0 2024-12-03T09:15:44,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742207_1383 (size=12454) 2024-12-03T09:15:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:44,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217404258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217404259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217404259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217404260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,372 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/a02c6dc8c7ae4886886e10a841c07857 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/a02c6dc8c7ae4886886e10a841c07857 2024-12-03T09:15:44,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217404370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,383 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#A#compaction#316 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:44,384 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/0f802489269541a3bce817566686a6fa is 175, key is test_row_0/A:col10/1733217342412/Put/seqid=0 2024-12-03T09:15:44,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217404379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217404379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217404380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,386 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/B of 52f05dcf6274e3c4c8ffbdf5058e1f61 into a02c6dc8c7ae4886886e10a841c07857(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,386 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:44,387 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/B, priority=12, startTime=1733217343945; duration=0sec 2024-12-03T09:15:44,387 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:44,387 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:B 2024-12-03T09:15:44,387 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:15:44,389 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49316 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:15:44,389 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/C is initiating minor compaction (all files) 2024-12-03T09:15:44,389 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/C in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:44,389 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/e3f367752a1944e382e37b7de0f9d49e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/53aeb059598a48708400e36cef72fb98, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/d5ff4fee56014d0c95498624d530028d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/1ff3ca6417184aefa06ce8bbcd5c47ab] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=48.2 K 2024-12-03T09:15:44,389 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting e3f367752a1944e382e37b7de0f9d49e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733217336819 2024-12-03T09:15:44,389 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 53aeb059598a48708400e36cef72fb98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733217337992 2024-12-03T09:15:44,390 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting d5ff4fee56014d0c95498624d530028d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733217340190 2024-12-03T09:15:44,390 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ff3ca6417184aefa06ce8bbcd5c47ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733217342411 2024-12-03T09:15:44,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742208_1384 (size=31903) 2024-12-03T09:15:44,411 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:44,411 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/06cf2c2fd5bf403fa0efb04fee31ad2f is 50, key is test_row_0/C:col10/1733217342412/Put/seqid=0 2024-12-03T09:15:44,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742209_1385 (size=12949) 2024-12-03T09:15:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T09:15:44,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:44,500 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ff0c3ab8d8d84c5dafea0fa1b59a2c3c_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ff0c3ab8d8d84c5dafea0fa1b59a2c3c_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:44,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/f78d1b36065a482493c65104e832fa74, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:44,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/f78d1b36065a482493c65104e832fa74 is 175, key is test_row_0/A:col10/1733217343085/Put/seqid=0 2024-12-03T09:15:44,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742210_1386 (size=31255) 2024-12-03T09:15:44,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217404589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217404589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217404589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217404589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,809 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/0f802489269541a3bce817566686a6fa as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/0f802489269541a3bce817566686a6fa 2024-12-03T09:15:44,813 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/A of 52f05dcf6274e3c4c8ffbdf5058e1f61 into 0f802489269541a3bce817566686a6fa(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:44,813 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:44,813 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/A, priority=12, startTime=1733217343945; duration=0sec 2024-12-03T09:15:44,813 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:44,813 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:A 2024-12-03T09:15:44,822 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/06cf2c2fd5bf403fa0efb04fee31ad2f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/06cf2c2fd5bf403fa0efb04fee31ad2f 2024-12-03T09:15:44,826 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/C of 52f05dcf6274e3c4c8ffbdf5058e1f61 into 06cf2c2fd5bf403fa0efb04fee31ad2f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:44,826 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:44,826 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/C, priority=12, startTime=1733217343946; duration=0sec 2024-12-03T09:15:44,826 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:44,826 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:C 2024-12-03T09:15:44,904 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/f78d1b36065a482493c65104e832fa74 2024-12-03T09:15:44,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217404900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217404900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217404900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/104f2d968690496e9eac795ebcff5e85 is 50, key is test_row_0/B:col10/1733217343085/Put/seqid=0 2024-12-03T09:15:44,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217404909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:44,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:44,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742211_1387 (size=12301) 2024-12-03T09:15:44,920 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/104f2d968690496e9eac795ebcff5e85 2024-12-03T09:15:44,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/6c548c8990fd4d0e992bd4a580fd4776 is 50, key is test_row_0/C:col10/1733217343085/Put/seqid=0 2024-12-03T09:15:44,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742212_1388 (size=12301) 2024-12-03T09:15:45,338 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/6c548c8990fd4d0e992bd4a580fd4776 2024-12-03T09:15:45,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/f78d1b36065a482493c65104e832fa74 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f78d1b36065a482493c65104e832fa74 2024-12-03T09:15:45,347 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f78d1b36065a482493c65104e832fa74, entries=150, sequenceid=290, filesize=30.5 K 2024-12-03T09:15:45,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/104f2d968690496e9eac795ebcff5e85 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/104f2d968690496e9eac795ebcff5e85 2024-12-03T09:15:45,352 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/104f2d968690496e9eac795ebcff5e85, entries=150, sequenceid=290, filesize=12.0 K 2024-12-03T09:15:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/6c548c8990fd4d0e992bd4a580fd4776 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/6c548c8990fd4d0e992bd4a580fd4776 2024-12-03T09:15:45,359 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/6c548c8990fd4d0e992bd4a580fd4776, entries=150, sequenceid=290, filesize=12.0 K 2024-12-03T09:15:45,360 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 1275ms, sequenceid=290, compaction requested=false 2024-12-03T09:15:45,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:45,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:45,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-03T09:15:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-03T09:15:45,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-03T09:15:45,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0390 sec 2024-12-03T09:15:45,365 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.0440 sec 2024-12-03T09:15:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:45,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T09:15:45,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:45,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:45,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:45,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:45,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:45,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:45,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038788ea5d3913436bbb22b63849d1ff28_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217344258/Put/seqid=0 2024-12-03T09:15:45,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742213_1389 (size=14994) 2024-12-03T09:15:45,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217405454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:45,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:45,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217405455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:45,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217405456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217405456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-03T09:15:45,469 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:45,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-03T09:15:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T09:15:45,471 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:45,473 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:45,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:45,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217405569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217405569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:45,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217405570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:45,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217405570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:45,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T09:15:45,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T09:15:45,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:45,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:45,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:45,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:45,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:45,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T09:15:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:45,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:45,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217405779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:45,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217405779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:45,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217405779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:45,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217405779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:45,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:45,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T09:15:45,840 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:45,844 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038788ea5d3913436bbb22b63849d1ff28_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038788ea5d3913436bbb22b63849d1ff28_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:45,844 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/412606bb44fa41dcb4a6b43e7283ad9d, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:45,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/412606bb44fa41dcb4a6b43e7283ad9d is 175, key is test_row_0/A:col10/1733217344258/Put/seqid=0 2024-12-03T09:15:45,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742214_1390 (size=39949) 2024-12-03T09:15:45,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T09:15:45,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:45,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:45,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:45,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:45,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:46,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T09:15:46,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:46,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:46,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:46,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:46,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:46,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:46,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217406089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217406089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T09:15:46,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217406099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:46,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217406099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:46,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T09:15:46,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:46,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:46,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:46,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:46,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:46,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:46,249 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/412606bb44fa41dcb4a6b43e7283ad9d 2024-12-03T09:15:46,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/3f30f164dcbe4afaafe87c152d7a5102 is 50, key is test_row_0/B:col10/1733217344258/Put/seqid=0 2024-12-03T09:15:46,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742215_1391 (size=12301) 2024-12-03T09:15:46,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T09:15:46,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:46,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:46,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:46,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:46,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:46,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T09:15:46,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:46,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:46,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:46,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:46,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:46,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:46,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46136 deadline: 1733217406599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:46,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:46,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46130 deadline: 1733217406599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:46,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T09:15:46,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46162 deadline: 1733217406610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46178 deadline: 1733217406610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:15:46,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/3f30f164dcbe4afaafe87c152d7a5102 2024-12-03T09:15:46,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/c27371bf19874efb8416185a5d9ccb15 is 50, key is test_row_0/C:col10/1733217344258/Put/seqid=0 2024-12-03T09:15:46,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742216_1392 (size=12301) 2024-12-03T09:15:46,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/c27371bf19874efb8416185a5d9ccb15 2024-12-03T09:15:46,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/412606bb44fa41dcb4a6b43e7283ad9d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/412606bb44fa41dcb4a6b43e7283ad9d 2024-12-03T09:15:46,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/412606bb44fa41dcb4a6b43e7283ad9d, entries=200, sequenceid=310, filesize=39.0 K 2024-12-03T09:15:46,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/3f30f164dcbe4afaafe87c152d7a5102 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/3f30f164dcbe4afaafe87c152d7a5102 2024-12-03T09:15:46,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/3f30f164dcbe4afaafe87c152d7a5102, entries=150, sequenceid=310, filesize=12.0 K 2024-12-03T09:15:46,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/c27371bf19874efb8416185a5d9ccb15 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/c27371bf19874efb8416185a5d9ccb15 2024-12-03T09:15:46,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/c27371bf19874efb8416185a5d9ccb15, entries=150, sequenceid=310, filesize=12.0 K 2024-12-03T09:15:46,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 1270ms, sequenceid=310, compaction requested=true 2024-12-03T09:15:46,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:46,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:46,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:46,693 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:46,696 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:46,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-03T09:15:46,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:46,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:46,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52f05dcf6274e3c4c8ffbdf5058e1f61:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:46,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:46,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:46,696 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T09:15:46,697 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:46,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:46,697 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/A is initiating minor compaction (all files) 2024-12-03T09:15:46,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:46,697 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:46,697 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/A in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:46,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:46,697 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/B is initiating minor compaction (all files) 2024-12-03T09:15:46,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:46,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:46,697 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/B in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:46,697 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/0f802489269541a3bce817566686a6fa, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f78d1b36065a482493c65104e832fa74, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/412606bb44fa41dcb4a6b43e7283ad9d] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=100.7 K 2024-12-03T09:15:46,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:46,697 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/a02c6dc8c7ae4886886e10a841c07857, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/104f2d968690496e9eac795ebcff5e85, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/3f30f164dcbe4afaafe87c152d7a5102] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=36.7 K 2024-12-03T09:15:46,697 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:46,697 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/0f802489269541a3bce817566686a6fa, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f78d1b36065a482493c65104e832fa74, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/412606bb44fa41dcb4a6b43e7283ad9d] 2024-12-03T09:15:46,699 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting a02c6dc8c7ae4886886e10a841c07857, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733217342411 2024-12-03T09:15:46,699 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f802489269541a3bce817566686a6fa, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733217342411 2024-12-03T09:15:46,700 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f78d1b36065a482493c65104e832fa74, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733217343068 2024-12-03T09:15:46,700 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 412606bb44fa41dcb4a6b43e7283ad9d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733217344249 2024-12-03T09:15:46,700 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 104f2d968690496e9eac795ebcff5e85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733217343068 2024-12-03T09:15:46,702 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f30f164dcbe4afaafe87c152d7a5102, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733217344249 2024-12-03T09:15:46,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b954a9146b684fe4afe20c46f91347fb_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217345455/Put/seqid=0 2024-12-03T09:15:46,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742217_1393 (size=12454) 2024-12-03T09:15:46,742 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#B#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:46,743 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/0fbd098dd99e4199b872ffdbd5eb7649 is 50, key is test_row_0/B:col10/1733217344258/Put/seqid=0 2024-12-03T09:15:46,750 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:46,757 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203ef7b720a06824a669b8da0818467b5c1_52f05dcf6274e3c4c8ffbdf5058e1f61 store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:46,759 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203ef7b720a06824a669b8da0818467b5c1_52f05dcf6274e3c4c8ffbdf5058e1f61, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:46,759 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ef7b720a06824a669b8da0818467b5c1_52f05dcf6274e3c4c8ffbdf5058e1f61 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:46,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742218_1394 (size=13051) 2024-12-03T09:15:46,785 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/0fbd098dd99e4199b872ffdbd5eb7649 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0fbd098dd99e4199b872ffdbd5eb7649 2024-12-03T09:15:46,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742219_1395 (size=4469) 2024-12-03T09:15:46,799 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/B of 52f05dcf6274e3c4c8ffbdf5058e1f61 into 0fbd098dd99e4199b872ffdbd5eb7649(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:46,799 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:46,799 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/B, priority=13, startTime=1733217346696; duration=0sec 2024-12-03T09:15:46,799 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:46,799 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:B 2024-12-03T09:15:46,799 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:46,799 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#A#compaction#326 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:46,800 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/fbe5e240af9c4472b084d6517ac17650 is 175, key is test_row_0/A:col10/1733217344258/Put/seqid=0 2024-12-03T09:15:46,802 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:46,803 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 52f05dcf6274e3c4c8ffbdf5058e1f61/C is initiating minor compaction (all files) 2024-12-03T09:15:46,803 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52f05dcf6274e3c4c8ffbdf5058e1f61/C in TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:46,803 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/06cf2c2fd5bf403fa0efb04fee31ad2f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/6c548c8990fd4d0e992bd4a580fd4776, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/c27371bf19874efb8416185a5d9ccb15] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp, totalSize=36.7 K 2024-12-03T09:15:46,803 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 06cf2c2fd5bf403fa0efb04fee31ad2f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733217342411 2024-12-03T09:15:46,803 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c548c8990fd4d0e992bd4a580fd4776, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733217343068 2024-12-03T09:15:46,804 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting c27371bf19874efb8416185a5d9ccb15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733217344249 2024-12-03T09:15:46,826 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52f05dcf6274e3c4c8ffbdf5058e1f61#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:46,827 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/1e742bdf6d124dc4ad594e6f6aa5ab21 is 50, key is test_row_0/C:col10/1733217344258/Put/seqid=0 2024-12-03T09:15:46,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742220_1396 (size=32005) 2024-12-03T09:15:46,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742221_1397 (size=13051) 2024-12-03T09:15:46,856 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/1e742bdf6d124dc4ad594e6f6aa5ab21 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/1e742bdf6d124dc4ad594e6f6aa5ab21 2024-12-03T09:15:46,865 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/C of 52f05dcf6274e3c4c8ffbdf5058e1f61 into 1e742bdf6d124dc4ad594e6f6aa5ab21(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:46,865 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:46,865 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/C, priority=13, startTime=1733217346696; duration=0sec 2024-12-03T09:15:46,865 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:46,865 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:C 2024-12-03T09:15:47,071 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/517764f5635540c1b5912a9b5647e9d9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/20af55685a2a4839807678dec6be7124, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f83a335c97514b0989a7390e64eeede8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae94047cdb0d4e159f9bca57f51a7597, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/058548458214480285c03315f7f29ea1, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f1210f07580d495dbfcadfff7fe1dde2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/212eb10b58ab4652b68eae994b992879, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/a55a728777d84e49a57de0bc63f4c691, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/e217fa0c39da49f38d8438ef13f0e524, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/d50c7b03a19648d0909820d69baa654e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/661ff56146394616a207a89cc37242da, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/6a3f8223f366410088e313e05d9a33cd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ab410b23bda245feabddcff5cbf5847e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/39cb7444bc2a402c90d76a387e3596f4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/d117de5cdd0c49778311778f17dc184f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/22ceb9d44f8e4db0a3cb927dca66b3e5, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae0044b8c9ec482b841dec305da75cb0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/42b4b5f682094d469a8d16e22f51fbfe, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/12ba0070c80e4e51bc73541c4daab0a4] to archive 2024-12-03T09:15:47,072 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:15:47,074 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/517764f5635540c1b5912a9b5647e9d9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/517764f5635540c1b5912a9b5647e9d9 2024-12-03T09:15:47,074 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/20af55685a2a4839807678dec6be7124 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/20af55685a2a4839807678dec6be7124 2024-12-03T09:15:47,075 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f83a335c97514b0989a7390e64eeede8 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f83a335c97514b0989a7390e64eeede8 2024-12-03T09:15:47,076 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae94047cdb0d4e159f9bca57f51a7597 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae94047cdb0d4e159f9bca57f51a7597 2024-12-03T09:15:47,077 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/058548458214480285c03315f7f29ea1 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/058548458214480285c03315f7f29ea1 2024-12-03T09:15:47,078 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f1210f07580d495dbfcadfff7fe1dde2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f1210f07580d495dbfcadfff7fe1dde2 2024-12-03T09:15:47,080 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/212eb10b58ab4652b68eae994b992879 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/212eb10b58ab4652b68eae994b992879 2024-12-03T09:15:47,080 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/a55a728777d84e49a57de0bc63f4c691 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/a55a728777d84e49a57de0bc63f4c691 2024-12-03T09:15:47,081 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/e217fa0c39da49f38d8438ef13f0e524 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/e217fa0c39da49f38d8438ef13f0e524 2024-12-03T09:15:47,082 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/d50c7b03a19648d0909820d69baa654e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/d50c7b03a19648d0909820d69baa654e 2024-12-03T09:15:47,083 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/661ff56146394616a207a89cc37242da to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/661ff56146394616a207a89cc37242da 2024-12-03T09:15:47,084 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/6a3f8223f366410088e313e05d9a33cd to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/6a3f8223f366410088e313e05d9a33cd 2024-12-03T09:15:47,087 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ab410b23bda245feabddcff5cbf5847e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ab410b23bda245feabddcff5cbf5847e 2024-12-03T09:15:47,088 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/39cb7444bc2a402c90d76a387e3596f4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/39cb7444bc2a402c90d76a387e3596f4 2024-12-03T09:15:47,089 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/d117de5cdd0c49778311778f17dc184f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/d117de5cdd0c49778311778f17dc184f 2024-12-03T09:15:47,090 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/22ceb9d44f8e4db0a3cb927dca66b3e5 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/22ceb9d44f8e4db0a3cb927dca66b3e5 2024-12-03T09:15:47,091 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae0044b8c9ec482b841dec305da75cb0 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/ae0044b8c9ec482b841dec305da75cb0 2024-12-03T09:15:47,092 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/42b4b5f682094d469a8d16e22f51fbfe to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/42b4b5f682094d469a8d16e22f51fbfe 2024-12-03T09:15:47,095 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/12ba0070c80e4e51bc73541c4daab0a4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/12ba0070c80e4e51bc73541c4daab0a4 2024-12-03T09:15:47,098 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/6b82c356a0f6452f9b19749520ce92e8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/32f62b6fc6f24424ae4008504617daec, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/c1a37515cdb94b6fb0644f8f880eb6c4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/f226258227e545fb86189bf23ac9e51f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/9780e48c2f694bf78a183bab3975ff77, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/d0a2d73bed2e4eb3b4571a69922a8ca3, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/343d766c2714427faececf63e249cf5a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/e65398c034aa48ad99a0f2cc0490d958, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/b0e37a0e811b47ac9ace3b56a9b9460b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/8fb5d6aefa254398a38ac3399b095130, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0b3064f74c9a40febe0016dfe53e20ca, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0ee47571021d4d719f17364f8f056c93, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/599a6b3a1eb640ebbaa32b7c5d67e888, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/1d3a1c6b670b4a13b2732cc269557507, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/31beb7cf90434162a0581c65c0e51a1b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/724c97c28af14d8b9a215e0ea04fb9c3, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/4b63ac71ac484b76ac531ddf7d4e648d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/4c7d83f2ba0742d48bda6e021c072909, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/a02c6dc8c7ae4886886e10a841c07857, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/b5f49e436c4c4095878dcf308505961e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/104f2d968690496e9eac795ebcff5e85, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/3f30f164dcbe4afaafe87c152d7a5102] to archive 2024-12-03T09:15:47,099 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:15:47,100 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/6b82c356a0f6452f9b19749520ce92e8 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/6b82c356a0f6452f9b19749520ce92e8 2024-12-03T09:15:47,101 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/32f62b6fc6f24424ae4008504617daec to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/32f62b6fc6f24424ae4008504617daec 2024-12-03T09:15:47,102 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/c1a37515cdb94b6fb0644f8f880eb6c4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/c1a37515cdb94b6fb0644f8f880eb6c4 2024-12-03T09:15:47,103 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/f226258227e545fb86189bf23ac9e51f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/f226258227e545fb86189bf23ac9e51f 2024-12-03T09:15:47,104 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/9780e48c2f694bf78a183bab3975ff77 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/9780e48c2f694bf78a183bab3975ff77 2024-12-03T09:15:47,105 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/d0a2d73bed2e4eb3b4571a69922a8ca3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/d0a2d73bed2e4eb3b4571a69922a8ca3 2024-12-03T09:15:47,106 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/343d766c2714427faececf63e249cf5a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/343d766c2714427faececf63e249cf5a 2024-12-03T09:15:47,106 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/e65398c034aa48ad99a0f2cc0490d958 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/e65398c034aa48ad99a0f2cc0490d958 2024-12-03T09:15:47,107 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/b0e37a0e811b47ac9ace3b56a9b9460b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/b0e37a0e811b47ac9ace3b56a9b9460b 2024-12-03T09:15:47,110 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/8fb5d6aefa254398a38ac3399b095130 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/8fb5d6aefa254398a38ac3399b095130 2024-12-03T09:15:47,111 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0b3064f74c9a40febe0016dfe53e20ca to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0b3064f74c9a40febe0016dfe53e20ca 2024-12-03T09:15:47,112 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0ee47571021d4d719f17364f8f056c93 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0ee47571021d4d719f17364f8f056c93 2024-12-03T09:15:47,113 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/599a6b3a1eb640ebbaa32b7c5d67e888 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/599a6b3a1eb640ebbaa32b7c5d67e888 2024-12-03T09:15:47,114 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/1d3a1c6b670b4a13b2732cc269557507 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/1d3a1c6b670b4a13b2732cc269557507 2024-12-03T09:15:47,116 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/31beb7cf90434162a0581c65c0e51a1b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/31beb7cf90434162a0581c65c0e51a1b 2024-12-03T09:15:47,117 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/724c97c28af14d8b9a215e0ea04fb9c3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/724c97c28af14d8b9a215e0ea04fb9c3 2024-12-03T09:15:47,118 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/4b63ac71ac484b76ac531ddf7d4e648d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/4b63ac71ac484b76ac531ddf7d4e648d 2024-12-03T09:15:47,119 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/4c7d83f2ba0742d48bda6e021c072909 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/4c7d83f2ba0742d48bda6e021c072909 2024-12-03T09:15:47,119 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/a02c6dc8c7ae4886886e10a841c07857 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/a02c6dc8c7ae4886886e10a841c07857 2024-12-03T09:15:47,120 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/b5f49e436c4c4095878dcf308505961e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/b5f49e436c4c4095878dcf308505961e 2024-12-03T09:15:47,121 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/104f2d968690496e9eac795ebcff5e85 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/104f2d968690496e9eac795ebcff5e85 2024-12-03T09:15:47,122 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/3f30f164dcbe4afaafe87c152d7a5102 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/3f30f164dcbe4afaafe87c152d7a5102 2024-12-03T09:15:47,127 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/9a1aa3c2216f4275beb459b4a3f2f82c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/2066dc028e904c208fc4197b4e08d23e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/396e486fc9534b17ae5f2fa204a5648c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/741181165170440fa356f849c241460f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/a8e316866d7d44519bafbe0687906559, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/fa9e40cb3a084dae97256542dcc4de5a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/0fd38db0f24c477785ac71dc5feedcba, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/33588709a5764ebe8e578440fa7cbdb2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/07af203a2acc4d21b19f3e90f391420c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/d1f8b337da174641b1927008f0f57db7, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/58c5f19cfc7f4c7bb8da71fb3556af7b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/a4a5d65e07cc4c9292f7c77fd737d6d2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/04d64c9f51394e8c9513e72d0afa7549, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/160bd8451a4948009a88ff539686bd52, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/e3f367752a1944e382e37b7de0f9d49e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/da77fd70046f4c3d84b9e61c0905aec8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/53aeb059598a48708400e36cef72fb98, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/d5ff4fee56014d0c95498624d530028d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/06cf2c2fd5bf403fa0efb04fee31ad2f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/1ff3ca6417184aefa06ce8bbcd5c47ab, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/6c548c8990fd4d0e992bd4a580fd4776, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/c27371bf19874efb8416185a5d9ccb15] to archive 2024-12-03T09:15:47,128 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:15:47,129 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/9a1aa3c2216f4275beb459b4a3f2f82c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/9a1aa3c2216f4275beb459b4a3f2f82c 2024-12-03T09:15:47,130 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/2066dc028e904c208fc4197b4e08d23e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/2066dc028e904c208fc4197b4e08d23e 2024-12-03T09:15:47,134 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/396e486fc9534b17ae5f2fa204a5648c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/396e486fc9534b17ae5f2fa204a5648c 2024-12-03T09:15:47,135 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/741181165170440fa356f849c241460f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/741181165170440fa356f849c241460f 2024-12-03T09:15:47,135 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/a8e316866d7d44519bafbe0687906559 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/a8e316866d7d44519bafbe0687906559 2024-12-03T09:15:47,136 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/fa9e40cb3a084dae97256542dcc4de5a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/fa9e40cb3a084dae97256542dcc4de5a 2024-12-03T09:15:47,138 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/0fd38db0f24c477785ac71dc5feedcba to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/0fd38db0f24c477785ac71dc5feedcba 2024-12-03T09:15:47,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:47,140 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/33588709a5764ebe8e578440fa7cbdb2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/33588709a5764ebe8e578440fa7cbdb2 2024-12-03T09:15:47,141 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/07af203a2acc4d21b19f3e90f391420c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/07af203a2acc4d21b19f3e90f391420c 2024-12-03T09:15:47,145 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b954a9146b684fe4afe20c46f91347fb_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b954a9146b684fe4afe20c46f91347fb_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:47,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/8141c232540c47da9abc2225f35ac175, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:47,146 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/d1f8b337da174641b1927008f0f57db7 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/d1f8b337da174641b1927008f0f57db7 2024-12-03T09:15:47,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/8141c232540c47da9abc2225f35ac175 is 175, key is test_row_0/A:col10/1733217345455/Put/seqid=0 2024-12-03T09:15:47,148 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/58c5f19cfc7f4c7bb8da71fb3556af7b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/58c5f19cfc7f4c7bb8da71fb3556af7b 2024-12-03T09:15:47,149 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/a4a5d65e07cc4c9292f7c77fd737d6d2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/a4a5d65e07cc4c9292f7c77fd737d6d2 2024-12-03T09:15:47,150 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/04d64c9f51394e8c9513e72d0afa7549 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/04d64c9f51394e8c9513e72d0afa7549 2024-12-03T09:15:47,151 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/160bd8451a4948009a88ff539686bd52 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/160bd8451a4948009a88ff539686bd52 2024-12-03T09:15:47,152 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/e3f367752a1944e382e37b7de0f9d49e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/e3f367752a1944e382e37b7de0f9d49e 2024-12-03T09:15:47,153 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/da77fd70046f4c3d84b9e61c0905aec8 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/da77fd70046f4c3d84b9e61c0905aec8 2024-12-03T09:15:47,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742222_1398 (size=31255) 2024-12-03T09:15:47,155 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/53aeb059598a48708400e36cef72fb98 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/53aeb059598a48708400e36cef72fb98 2024-12-03T09:15:47,155 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/8141c232540c47da9abc2225f35ac175 2024-12-03T09:15:47,159 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/d5ff4fee56014d0c95498624d530028d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/d5ff4fee56014d0c95498624d530028d 2024-12-03T09:15:47,165 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/06cf2c2fd5bf403fa0efb04fee31ad2f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/06cf2c2fd5bf403fa0efb04fee31ad2f 2024-12-03T09:15:47,168 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/1ff3ca6417184aefa06ce8bbcd5c47ab to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/1ff3ca6417184aefa06ce8bbcd5c47ab 2024-12-03T09:15:47,169 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/6c548c8990fd4d0e992bd4a580fd4776 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/6c548c8990fd4d0e992bd4a580fd4776 2024-12-03T09:15:47,170 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/c27371bf19874efb8416185a5d9ccb15 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/c27371bf19874efb8416185a5d9ccb15 2024-12-03T09:15:47,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/bf547657e0da4e6f98cd9377258ec331 is 50, key is test_row_0/B:col10/1733217345455/Put/seqid=0 2024-12-03T09:15:47,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742223_1399 (size=12301) 2024-12-03T09:15:47,173 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/bf547657e0da4e6f98cd9377258ec331 2024-12-03T09:15:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/61708f2ee4c245599c03d0d15c030de1 is 50, key is test_row_0/C:col10/1733217345455/Put/seqid=0 2024-12-03T09:15:47,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742224_1400 (size=12301) 2024-12-03T09:15:47,238 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/fbe5e240af9c4472b084d6517ac17650 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/fbe5e240af9c4472b084d6517ac17650 2024-12-03T09:15:47,246 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52f05dcf6274e3c4c8ffbdf5058e1f61/A of 52f05dcf6274e3c4c8ffbdf5058e1f61 into fbe5e240af9c4472b084d6517ac17650(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:47,246 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:47,246 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., storeName=52f05dcf6274e3c4c8ffbdf5058e1f61/A, priority=13, startTime=1733217346693; duration=0sec 2024-12-03T09:15:47,246 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:47,246 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52f05dcf6274e3c4c8ffbdf5058e1f61:A 2024-12-03T09:15:47,249 INFO [Thread-1422 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1422. 2024-12-03T09:15:47,249 DEBUG [Thread-1422 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:47,249 DEBUG [Thread-1422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,249 INFO [Thread-1424 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1424. 2024-12-03T09:15:47,249 DEBUG [Thread-1424 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:47,249 DEBUG [Thread-1424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,249 DEBUG [Thread-1422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,249 DEBUG [Thread-1424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:47,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:47,250 INFO [Thread-1420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1420. 2024-12-03T09:15:47,250 DEBUG [Thread-1420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:47,250 DEBUG [Thread-1420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,250 DEBUG [Thread-1420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,250 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:47,257 INFO [Thread-1428 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1428. 2024-12-03T09:15:47,257 INFO [Thread-1426 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1426. 2024-12-03T09:15:47,257 DEBUG [Thread-1428 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:47,257 DEBUG [Thread-1426 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:47,257 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,257 DEBUG [Thread-1426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,257 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,257 DEBUG [Thread-1426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:47,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:47,588 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/61708f2ee4c245599c03d0d15c030de1 2024-12-03T09:15:47,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/8141c232540c47da9abc2225f35ac175 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/8141c232540c47da9abc2225f35ac175 2024-12-03T09:15:47,601 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/8141c232540c47da9abc2225f35ac175, entries=150, sequenceid=325, filesize=30.5 K 2024-12-03T09:15:47,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/bf547657e0da4e6f98cd9377258ec331 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/bf547657e0da4e6f98cd9377258ec331 2024-12-03T09:15:47,607 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/bf547657e0da4e6f98cd9377258ec331, entries=150, sequenceid=325, filesize=12.0 K 2024-12-03T09:15:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/61708f2ee4c245599c03d0d15c030de1 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/61708f2ee4c245599c03d0d15c030de1 2024-12-03T09:15:47,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. as already flushing 2024-12-03T09:15:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:47,611 INFO [Thread-1411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1411. 2024-12-03T09:15:47,611 DEBUG [Thread-1411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:47,611 DEBUG [Thread-1411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,611 DEBUG [Thread-1411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:47,613 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/61708f2ee4c245599c03d0d15c030de1, entries=150, sequenceid=325, filesize=12.0 K 2024-12-03T09:15:47,613 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=6.71 KB/6870 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 917ms, sequenceid=325, compaction requested=false 2024-12-03T09:15:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: 2024-12-03T09:15:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:47,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-03T09:15:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-03T09:15:47,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-03T09:15:47,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1410 sec 2024-12-03T09:15:47,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.1470 sec 2024-12-03T09:15:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T09:15:47,619 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:47,620 INFO [Thread-1413 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1413. 2024-12-03T09:15:47,620 DEBUG [Thread-1413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:47,620 DEBUG [Thread-1413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,621 DEBUG [Thread-1413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:47,630 INFO [Thread-1415 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1415. 2024-12-03T09:15:47,630 DEBUG [Thread-1415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:47,630 INFO [Thread-1409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1409. 2024-12-03T09:15:47,630 DEBUG [Thread-1415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,630 DEBUG [Thread-1409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:47,631 DEBUG [Thread-1409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,631 DEBUG [Thread-1415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,631 DEBUG [Thread-1409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:47,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:47,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:52,501 INFO [Thread-1417 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1417. 2024-12-03T09:15:52,501 DEBUG [Thread-1417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:15:52,501 DEBUG [Thread-1417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:52,501 DEBUG [Thread-1417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:52,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1489 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4467 rows 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1497 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4491 rows 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1492 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4473 rows 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1490 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4469 rows 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1495 2024-12-03T09:15:52,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4485 rows 2024-12-03T09:15:52,502 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T09:15:52,502 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:15:52,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:52,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:52,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:15:52,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T09:15:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T09:15:52,508 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217352508"}]},"ts":"1733217352508"} 2024-12-03T09:15:52,510 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T09:15:52,510 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T09:15:52,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T09:15:52,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, UNASSIGN}] 2024-12-03T09:15:52,513 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, UNASSIGN 2024-12-03T09:15:52,514 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=52f05dcf6274e3c4c8ffbdf5058e1f61, regionState=CLOSING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:52,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, UNASSIGN because future has completed 2024-12-03T09:15:52,516 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T09:15:52,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:15:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T09:15:52,670 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:52,670 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T09:15:52,670 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 52f05dcf6274e3c4c8ffbdf5058e1f61, disabling compactions & flushes 2024-12-03T09:15:52,670 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:52,670 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:52,671 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. after waiting 0 ms 2024-12-03T09:15:52,671 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:52,671 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T09:15:52,671 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=A 2024-12-03T09:15:52,671 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:52,672 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=B 2024-12-03T09:15:52,672 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:52,672 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 52f05dcf6274e3c4c8ffbdf5058e1f61, store=C 2024-12-03T09:15:52,672 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:52,679 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d89335cf41884adb948bc45cebd27daf_52f05dcf6274e3c4c8ffbdf5058e1f61 is 50, key is test_row_0/A:col10/1733217352499/Put/seqid=0 2024-12-03T09:15:52,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742225_1401 (size=12454) 2024-12-03T09:15:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T09:15:53,084 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:53,090 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d89335cf41884adb948bc45cebd27daf_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d89335cf41884adb948bc45cebd27daf_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:53,091 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/3064c70383ae49338410235259f6de26, store: [table=TestAcidGuarantees family=A region=52f05dcf6274e3c4c8ffbdf5058e1f61] 2024-12-03T09:15:53,092 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/3064c70383ae49338410235259f6de26 is 175, key is test_row_0/A:col10/1733217352499/Put/seqid=0 2024-12-03T09:15:53,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742226_1402 (size=31255) 2024-12-03T09:15:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T09:15:53,495 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/3064c70383ae49338410235259f6de26 2024-12-03T09:15:53,501 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/606c8ab7e1854d00a189f6021157fea1 is 50, key is test_row_0/B:col10/1733217352499/Put/seqid=0 2024-12-03T09:15:53,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742227_1403 (size=12301) 2024-12-03T09:15:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T09:15:53,906 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/606c8ab7e1854d00a189f6021157fea1 2024-12-03T09:15:53,918 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/b5ca880b0ae746f0997ef86b9a00ff0f is 50, key is test_row_0/C:col10/1733217352499/Put/seqid=0 2024-12-03T09:15:53,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742228_1404 (size=12301) 2024-12-03T09:15:54,322 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/b5ca880b0ae746f0997ef86b9a00ff0f 2024-12-03T09:15:54,326 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/A/3064c70383ae49338410235259f6de26 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/3064c70383ae49338410235259f6de26 2024-12-03T09:15:54,330 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/3064c70383ae49338410235259f6de26, entries=150, sequenceid=336, filesize=30.5 K 2024-12-03T09:15:54,330 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/B/606c8ab7e1854d00a189f6021157fea1 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/606c8ab7e1854d00a189f6021157fea1 2024-12-03T09:15:54,335 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/606c8ab7e1854d00a189f6021157fea1, entries=150, sequenceid=336, filesize=12.0 K 2024-12-03T09:15:54,336 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/.tmp/C/b5ca880b0ae746f0997ef86b9a00ff0f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/b5ca880b0ae746f0997ef86b9a00ff0f 2024-12-03T09:15:54,341 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/b5ca880b0ae746f0997ef86b9a00ff0f, entries=150, sequenceid=336, filesize=12.0 K 2024-12-03T09:15:54,342 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 1671ms, sequenceid=336, compaction requested=true 2024-12-03T09:15:54,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/0f802489269541a3bce817566686a6fa, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f78d1b36065a482493c65104e832fa74, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/412606bb44fa41dcb4a6b43e7283ad9d] to archive 2024-12-03T09:15:54,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:15:54,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/0f802489269541a3bce817566686a6fa to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/0f802489269541a3bce817566686a6fa 2024-12-03T09:15:54,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f78d1b36065a482493c65104e832fa74 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/f78d1b36065a482493c65104e832fa74 2024-12-03T09:15:54,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/412606bb44fa41dcb4a6b43e7283ad9d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/412606bb44fa41dcb4a6b43e7283ad9d 2024-12-03T09:15:54,351 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/recovered.edits/339.seqid, newMaxSeqId=339, maxSeqId=4 2024-12-03T09:15:54,352 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. 2024-12-03T09:15:54,352 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 52f05dcf6274e3c4c8ffbdf5058e1f61: Waiting for close lock at 1733217352670Running coprocessor pre-close hooks at 1733217352670Disabling compacts and flushes for region at 1733217352670Disabling writes for close at 1733217352671 (+1 ms)Obtaining lock to block concurrent updates at 1733217352671Preparing flush snapshotting stores in 52f05dcf6274e3c4c8ffbdf5058e1f61 at 1733217352671Finished memstore snapshotting TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733217352672 (+1 ms)Flushing stores of TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61. at 1733217352673 (+1 ms)Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61/A: creating writer at 1733217352673Flushing mob file 52f05dcf6274e3c4c8ffbdf5058e1f61/A: appending metadata at 1733217352679 (+6 ms)Flushing mob file 52f05dcf6274e3c4c8ffbdf5058e1f61/A: closing flushed file at 1733217352679Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61/A: appending metadata at 1733217353091 (+412 ms)Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61/A: closing flushed file at 1733217353091Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61/B: creating writer at 1733217353499 (+408 ms)Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61/B: appending metadata at 1733217353501 (+2 ms)Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61/B: closing flushed file at 1733217353501Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61/C: creating writer at 1733217353914 (+413 ms)Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61/C: appending metadata at 1733217353918 (+4 ms)Flushing 52f05dcf6274e3c4c8ffbdf5058e1f61/C: closing flushed file at 1733217353918Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dc25ff9: reopening flushed file at 1733217354326 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8ca67d: reopening flushed file at 1733217354330 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12b13350: reopening flushed file at 1733217354335 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 52f05dcf6274e3c4c8ffbdf5058e1f61 in 1671ms, sequenceid=336, compaction requested=true at 1733217354342 (+7 ms)Writing region close event to WAL at 1733217354349 (+7 ms)Running coprocessor post-close hooks at 1733217354352 (+3 ms)Closed at 1733217354352 2024-12-03T09:15:54,354 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,354 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=52f05dcf6274e3c4c8ffbdf5058e1f61, regionState=CLOSED 2024-12-03T09:15:54,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:15:54,357 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-03T09:15:54,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 52f05dcf6274e3c4c8ffbdf5058e1f61, server=699c9baf3d29,42271,1733217224446 in 1.8400 sec 2024-12-03T09:15:54,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-03T09:15:54,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52f05dcf6274e3c4c8ffbdf5058e1f61, UNASSIGN in 1.8460 sec 2024-12-03T09:15:54,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-03T09:15:54,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8480 sec 2024-12-03T09:15:54,361 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217354361"}]},"ts":"1733217354361"} 2024-12-03T09:15:54,362 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T09:15:54,363 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T09:15:54,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8590 sec 2024-12-03T09:15:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T09:15:54,660 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:54,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T09:15:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:54,664 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T09:15:54,666 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:54,671 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,675 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/recovered.edits] 2024-12-03T09:15:54,721 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/3064c70383ae49338410235259f6de26 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/3064c70383ae49338410235259f6de26 2024-12-03T09:15:54,723 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/8141c232540c47da9abc2225f35ac175 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/8141c232540c47da9abc2225f35ac175 2024-12-03T09:15:54,724 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/fbe5e240af9c4472b084d6517ac17650 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/A/fbe5e240af9c4472b084d6517ac17650 2024-12-03T09:15:54,726 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0fbd098dd99e4199b872ffdbd5eb7649 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/0fbd098dd99e4199b872ffdbd5eb7649 2024-12-03T09:15:54,727 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/606c8ab7e1854d00a189f6021157fea1 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/606c8ab7e1854d00a189f6021157fea1 2024-12-03T09:15:54,728 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/bf547657e0da4e6f98cd9377258ec331 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/B/bf547657e0da4e6f98cd9377258ec331 2024-12-03T09:15:54,731 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/1e742bdf6d124dc4ad594e6f6aa5ab21 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/1e742bdf6d124dc4ad594e6f6aa5ab21 2024-12-03T09:15:54,732 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/61708f2ee4c245599c03d0d15c030de1 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/61708f2ee4c245599c03d0d15c030de1 2024-12-03T09:15:54,734 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/b5ca880b0ae746f0997ef86b9a00ff0f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/C/b5ca880b0ae746f0997ef86b9a00ff0f 2024-12-03T09:15:54,737 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/recovered.edits/339.seqid to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61/recovered.edits/339.seqid 2024-12-03T09:15:54,738 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,738 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T09:15:54,739 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T09:15:54,740 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T09:15:54,744 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030a147d4ffc1e4f529910bde31fb57f2d_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030a147d4ffc1e4f529910bde31fb57f2d_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,746 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032cc61765b1364e008f40912783d7c92b_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032cc61765b1364e008f40912783d7c92b_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,748 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033a7d7c83d8f847e3858573ee682849f5_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033a7d7c83d8f847e3858573ee682849f5_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,750 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034fa042d112154840b03482eaa4d1cac5_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034fa042d112154840b03482eaa4d1cac5_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,751 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120371e7d3b774414d0cb685f9b6c3b417a4_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120371e7d3b774414d0cb685f9b6c3b417a4_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,752 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037cd638b186fc480d82f4f0b06fe875b1_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037cd638b186fc480d82f4f0b06fe875b1_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,754 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038788ea5d3913436bbb22b63849d1ff28_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038788ea5d3913436bbb22b63849d1ff28_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,755 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039f93709e364e4881902704a066f5d570_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039f93709e364e4881902704a066f5d570_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,756 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a1bbb2da771b4e93aeb21d90d69dfbab_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a1bbb2da771b4e93aeb21d90d69dfbab_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,757 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ad69cb471ff045deae6e1a3abbf251a2_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ad69cb471ff045deae6e1a3abbf251a2_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,759 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b954a9146b684fe4afe20c46f91347fb_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b954a9146b684fe4afe20c46f91347fb_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,760 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d64f7b8088d74ccab62d7582cc758c2f_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d64f7b8088d74ccab62d7582cc758c2f_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,761 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d89335cf41884adb948bc45cebd27daf_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d89335cf41884adb948bc45cebd27daf_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,763 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ed0341840f4e4afb8ea1410733d0a2fb_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ed0341840f4e4afb8ea1410733d0a2fb_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,764 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f06051b4f10749aebfd2f3060a576029_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f06051b4f10749aebfd2f3060a576029_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,765 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f42551ffc8d14a5a9af677c96b0762f6_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f42551ffc8d14a5a9af677c96b0762f6_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,766 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f9b8eacf753046ceb30d7b216793baad_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203f9b8eacf753046ceb30d7b216793baad_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,767 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ff0c3ab8d8d84c5dafea0fa1b59a2c3c_52f05dcf6274e3c4c8ffbdf5058e1f61 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ff0c3ab8d8d84c5dafea0fa1b59a2c3c_52f05dcf6274e3c4c8ffbdf5058e1f61 2024-12-03T09:15:54,767 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T09:15:54,769 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T09:15:54,771 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T09:15:54,772 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T09:15:54,773 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:54,773 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T09:15:54,773 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733217354773"}]},"ts":"9223372036854775807"} 2024-12-03T09:15:54,775 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T09:15:54,775 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 52f05dcf6274e3c4c8ffbdf5058e1f61, NAME => 'TestAcidGuarantees,,1733217324092.52f05dcf6274e3c4c8ffbdf5058e1f61.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T09:15:54,775 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T09:15:54,775 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733217354775"}]},"ts":"9223372036854775807"} 2024-12-03T09:15:54,777 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T09:15:54,778 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:54,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 117 msec 2024-12-03T09:15:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T09:15:54,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T09:15:54,986 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:54,995 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 246) - Thread LEAK? -, OpenFileDescriptor=449 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=387 (was 374) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=13326 (was 12808) - AvailableMemoryMB LEAK? - 2024-12-03T09:15:55,004 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=387, ProcessCount=11, AvailableMemoryMB=13325 2024-12-03T09:15:55,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T09:15:55,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T09:15:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T09:15:55,008 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T09:15:55,008 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:55,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-12-03T09:15:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-03T09:15:55,009 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T09:15:55,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742229_1405 (size=1061) 2024-12-03T09:15:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-03T09:15:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-03T09:15:55,418 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09 2024-12-03T09:15:55,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742230_1406 (size=53) 2024-12-03T09:15:55,457 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:15:55,457 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 427f6f3f7c0d84ddc6f684fba93003f4, disabling compactions & flushes 2024-12-03T09:15:55,457 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:55,458 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:55,458 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. after waiting 0 ms 2024-12-03T09:15:55,458 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:55,458 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:55,458 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 427f6f3f7c0d84ddc6f684fba93003f4: Waiting for close lock at 1733217355457Disabling compacts and flushes for region at 1733217355457Disabling writes for close at 1733217355458 (+1 ms)Writing region close event to WAL at 1733217355458Closed at 1733217355458 2024-12-03T09:15:55,459 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T09:15:55,459 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733217355459"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733217355459"}]},"ts":"1733217355459"} 2024-12-03T09:15:55,460 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T09:15:55,461 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T09:15:55,462 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217355461"}]},"ts":"1733217355461"} 2024-12-03T09:15:55,464 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T09:15:55,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=427f6f3f7c0d84ddc6f684fba93003f4, ASSIGN}] 2024-12-03T09:15:55,465 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=427f6f3f7c0d84ddc6f684fba93003f4, ASSIGN 2024-12-03T09:15:55,466 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=427f6f3f7c0d84ddc6f684fba93003f4, ASSIGN; state=OFFLINE, location=699c9baf3d29,42271,1733217224446; forceNewPlan=false, retain=false 2024-12-03T09:15:55,617 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=427f6f3f7c0d84ddc6f684fba93003f4, regionState=OPENING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:55,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=427f6f3f7c0d84ddc6f684fba93003f4, ASSIGN because future has completed 2024-12-03T09:15:55,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:15:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-03T09:15:55,775 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:55,776 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:15:55,777 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:55,777 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:15:55,777 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:55,777 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:55,778 INFO [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:55,780 INFO [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:15:55,780 INFO [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 427f6f3f7c0d84ddc6f684fba93003f4 columnFamilyName A 2024-12-03T09:15:55,780 DEBUG [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:55,781 INFO [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] regionserver.HStore(327): Store=427f6f3f7c0d84ddc6f684fba93003f4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:15:55,781 INFO [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:55,782 INFO [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:15:55,783 INFO [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 427f6f3f7c0d84ddc6f684fba93003f4 columnFamilyName B 2024-12-03T09:15:55,783 DEBUG [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:55,783 INFO [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] regionserver.HStore(327): Store=427f6f3f7c0d84ddc6f684fba93003f4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:15:55,784 INFO [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:55,785 INFO [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:15:55,786 INFO [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 427f6f3f7c0d84ddc6f684fba93003f4 columnFamilyName C 2024-12-03T09:15:55,786 DEBUG [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:15:55,787 INFO [StoreOpener-427f6f3f7c0d84ddc6f684fba93003f4-1 {}] regionserver.HStore(327): Store=427f6f3f7c0d84ddc6f684fba93003f4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:15:55,787 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:55,787 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:55,788 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:55,788 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:55,790 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:55,791 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:55,794 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T09:15:55,796 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:55,800 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:15:55,800 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 427f6f3f7c0d84ddc6f684fba93003f4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66285820, jitterRate=-0.01226431131362915}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T09:15:55,800 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:55,801 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 427f6f3f7c0d84ddc6f684fba93003f4: Running coprocessor pre-open hook at 1733217355777Writing region info on filesystem at 1733217355777Initializing all the Stores at 1733217355778 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217355778Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217355778Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217355778Cleaning up temporary data from old regions at 1733217355791 (+13 ms)Running coprocessor post-open hooks at 1733217355800 (+9 ms)Region opened successfully at 1733217355801 (+1 ms) 2024-12-03T09:15:55,802 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., pid=118, masterSystemTime=1733217355772 2024-12-03T09:15:55,804 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:55,804 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:55,804 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=427f6f3f7c0d84ddc6f684fba93003f4, regionState=OPEN, openSeqNum=2, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:55,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:15:55,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-03T09:15:55,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 in 188 msec 2024-12-03T09:15:55,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-12-03T09:15:55,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=427f6f3f7c0d84ddc6f684fba93003f4, ASSIGN in 348 msec 2024-12-03T09:15:55,819 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T09:15:55,819 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217355819"}]},"ts":"1733217355819"} 2024-12-03T09:15:55,821 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T09:15:55,822 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T09:15:55,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 816 msec 2024-12-03T09:15:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-03T09:15:56,149 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:15:56,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61fea7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,150 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:56,150 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:56,151 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:56,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:56,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:56,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@756d5016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:56,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:56,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:56,153 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:56,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f58ad19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,154 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:56,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:56,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:56,157 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T09:15:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 699c9baf3d29,35781,1733217223715 2024-12-03T09:15:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@35de392d 2024-12-03T09:15:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T09:15:56,159 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T09:15:56,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@152505ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:56,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:56,161 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c55b312, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:56,163 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8edaca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60915904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,164 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:56,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:56,165 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:56,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:56,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:56,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586d5e7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:56,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:56,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:56,167 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d73d035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc3b479, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,168 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:56,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:56,169 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50027af8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:56,171 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d7a34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f3bcaf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,172 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:56,172 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:56,173 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1b72e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:56,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:56,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:56,175 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:56,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@caf284f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27f366e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,176 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:56,176 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:56,177 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:56,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:56,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60ecf499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:56,179 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72585fe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179662d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,180 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:56,180 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:56,181 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:56,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:56,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:56,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59712dae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:56,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:56,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:56,183 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa7f5ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be9379c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:56,185 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:56,185 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dbf7439, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:56,186 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:56,188 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9a7060, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a37eaef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:56,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:56,190 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:56,190 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f9fcfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:56,192 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:56,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@618abe73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d5681bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:56,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:56,194 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7563ae0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:56,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:56,195 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:56,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7753c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61bcd96c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:15:56,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:15:56,196 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:15:56,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:15:56,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:15:56,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6af5e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:15:56,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:15:56,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:15:56,197 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:15:56,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@629a4f34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:15:56,198 DEBUG [Thread-1865 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:56,198 DEBUG [Thread-1863 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:56,198 DEBUG [Thread-1848 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:56,198 DEBUG [Thread-1850 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:56,198 DEBUG [Thread-1856 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:56,198 DEBUG [Thread-1854 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:56,199 DEBUG [Thread-1852 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:56,199 DEBUG [Thread-1859 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:56,200 DEBUG [Thread-1867 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:56,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:56,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:15:56,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:56,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:56,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:56,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:56,201 DEBUG [Thread-1861 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:15:56,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:56,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:56,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:56,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:15:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:15:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-03T09:15:56,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:56,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:15:56,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T09:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:15:56,204 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:15:56,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:15:56,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:15:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:15:56,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:15:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:15:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:15:56,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2] 2024-12-03T09:15:56,207 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:15:56,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:15:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:56,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:15:56,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:15:56,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:56,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:15:56,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:56,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:15:56,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:56,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/1ca8cb46cfc247e9af8b5e231d021acb is 50, key is test_row_0/A:col10/1733217356211/Put/seqid=0 2024-12-03T09:15:56,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217416236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217416236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217416239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217356222, max timestamp=1733217356242Num uniques 859; 2024-12-03T09:15:56,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:15:56,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217356222, max timestamp=1733217356242Num uniques 859; 2024-12-03T09:15:56,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:15:56,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217356222, max timestamp=1733217356242Num uniques 859; 2024-12-03T09:15:56,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:15:56,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742231_1407 (size=14341) 2024-12-03T09:15:56,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217416282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217416283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T09:15:56,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217416349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217416349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217416349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T09:15:56,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:56,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:15:56,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:56,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217416390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217416399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T09:15:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:15:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:56,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T09:15:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217416559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217416569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217416569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217416599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217416609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/1ca8cb46cfc247e9af8b5e231d021acb 2024-12-03T09:15:56,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T09:15:56,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:56,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:15:56,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:56,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/76f029183f8a4d4b8f824442040a760a is 50, key is test_row_0/B:col10/1733217356211/Put/seqid=0 2024-12-03T09:15:56,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742232_1408 (size=12001) 2024-12-03T09:15:56,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T09:15:56,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:56,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:15:56,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:56,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T09:15:56,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217416869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217416879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217416879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217416909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217416919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:56,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:56,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:56,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T09:15:56,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:56,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:15:56,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:56,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:57,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/76f029183f8a4d4b8f824442040a760a 2024-12-03T09:15:57,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/4b1348939af449529c4eb76c682cf7f3 is 50, key is test_row_0/C:col10/1733217356211/Put/seqid=0 2024-12-03T09:15:57,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742233_1409 (size=12001) 2024-12-03T09:15:57,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T09:15:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:15:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:57,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:57,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:57,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T09:15:57,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:57,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:15:57,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:57,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:57,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T09:15:57,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217417379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:57,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:57,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:57,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:57,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217417390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:57,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217417390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:57,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217417419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:57,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T09:15:57,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:57,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:15:57,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:57,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:57,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:15:57,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217417439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:57,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:57,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/4b1348939af449529c4eb76c682cf7f3 2024-12-03T09:15:57,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/1ca8cb46cfc247e9af8b5e231d021acb as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/1ca8cb46cfc247e9af8b5e231d021acb 2024-12-03T09:15:57,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/1ca8cb46cfc247e9af8b5e231d021acb, entries=200, sequenceid=13, filesize=14.0 K 2024-12-03T09:15:57,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/76f029183f8a4d4b8f824442040a760a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/76f029183f8a4d4b8f824442040a760a 2024-12-03T09:15:57,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/76f029183f8a4d4b8f824442040a760a, entries=150, sequenceid=13, filesize=11.7 K 2024-12-03T09:15:57,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/4b1348939af449529c4eb76c682cf7f3 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/4b1348939af449529c4eb76c682cf7f3 2024-12-03T09:15:57,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/4b1348939af449529c4eb76c682cf7f3, entries=150, sequenceid=13, filesize=11.7 K 2024-12-03T09:15:57,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 427f6f3f7c0d84ddc6f684fba93003f4 in 1300ms, sequenceid=13, compaction requested=false 2024-12-03T09:15:57,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:15:57,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T09:15:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:57,588 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-03T09:15:57,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:15:57,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:57,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:15:57,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:57,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:15:57,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:15:57,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/3681826c492a4016876b0b467aef27b6 is 50, key is test_row_0/A:col10/1733217356281/Put/seqid=0 2024-12-03T09:15:57,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742234_1410 (size=12001) 2024-12-03T09:15:57,997 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/3681826c492a4016876b0b467aef27b6 2024-12-03T09:15:58,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/e9892308d2784422a3bfc10d805d8f92 is 50, key is test_row_0/B:col10/1733217356281/Put/seqid=0 2024-12-03T09:15:58,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742235_1411 (size=12001) 2024-12-03T09:15:58,014 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/e9892308d2784422a3bfc10d805d8f92 2024-12-03T09:15:58,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/416004a2be904b83b38b4acb8cb84b92 is 50, key is test_row_0/C:col10/1733217356281/Put/seqid=0 2024-12-03T09:15:58,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742236_1412 (size=12001) 2024-12-03T09:15:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T09:15:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:58,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:15:58,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217418411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217418412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217418414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217418429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,430 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 427f6f3f7c0d84ddc6f684fba93003f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:15:58,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,438 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/416004a2be904b83b38b4acb8cb84b92 2024-12-03T09:15:58,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/3681826c492a4016876b0b467aef27b6 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/3681826c492a4016876b0b467aef27b6 2024-12-03T09:15:58,446 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/3681826c492a4016876b0b467aef27b6, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T09:15:58,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/e9892308d2784422a3bfc10d805d8f92 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e9892308d2784422a3bfc10d805d8f92 2024-12-03T09:15:58,449 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e9892308d2784422a3bfc10d805d8f92, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T09:15:58,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/416004a2be904b83b38b4acb8cb84b92 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/416004a2be904b83b38b4acb8cb84b92 2024-12-03T09:15:58,453 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/416004a2be904b83b38b4acb8cb84b92, entries=150, sequenceid=47, filesize=11.7 K 2024-12-03T09:15:58,454 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 427f6f3f7c0d84ddc6f684fba93003f4 in 867ms, sequenceid=47, compaction requested=false 2024-12-03T09:15:58,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:15:58,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:58,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-03T09:15:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-03T09:15:58,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-03T09:15:58,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2490 sec 2024-12-03T09:15:58,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.2560 sec 2024-12-03T09:15:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:58,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T09:15:58,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:15:58,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:58,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:15:58,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:58,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:15:58,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:58,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/ce7357a32bd842078b701df5c66fc582 is 50, key is test_row_0/A:col10/1733217358413/Put/seqid=0 2024-12-03T09:15:58,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742237_1413 (size=14341) 2024-12-03T09:15:58,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217418520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217418526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217418529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217418529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217418629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217418639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217418739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217418739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217418839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217418849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:58,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:58,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:58,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/ce7357a32bd842078b701df5c66fc582 2024-12-03T09:15:58,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/3c09d7ed2e124e84a4e272d9951f8288 is 50, key is test_row_0/B:col10/1733217358413/Put/seqid=0 2024-12-03T09:15:58,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742238_1414 (size=12001) 2024-12-03T09:15:59,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217419049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217419049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:59,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217419159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:59,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217419159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:59,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/3c09d7ed2e124e84a4e272d9951f8288 2024-12-03T09:15:59,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c085a8b9ba6b4b178d6a6579789bf416 is 50, key is test_row_0/C:col10/1733217358413/Put/seqid=0 2024-12-03T09:15:59,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742239_1415 (size=12001) 2024-12-03T09:15:59,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c085a8b9ba6b4b178d6a6579789bf416 2024-12-03T09:15:59,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/ce7357a32bd842078b701df5c66fc582 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/ce7357a32bd842078b701df5c66fc582 2024-12-03T09:15:59,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/ce7357a32bd842078b701df5c66fc582, entries=200, sequenceid=60, filesize=14.0 K 2024-12-03T09:15:59,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/3c09d7ed2e124e84a4e272d9951f8288 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/3c09d7ed2e124e84a4e272d9951f8288 2024-12-03T09:15:59,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/3c09d7ed2e124e84a4e272d9951f8288, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T09:15:59,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c085a8b9ba6b4b178d6a6579789bf416 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c085a8b9ba6b4b178d6a6579789bf416 2024-12-03T09:15:59,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c085a8b9ba6b4b178d6a6579789bf416, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T09:15:59,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 427f6f3f7c0d84ddc6f684fba93003f4 in 885ms, sequenceid=60, compaction requested=true 2024-12-03T09:15:59,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:15:59,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:15:59,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:59,348 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:59,348 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:59,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:15:59,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:59,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:15:59,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:59,352 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:59,352 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:59,352 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/A is initiating minor compaction (all files) 2024-12-03T09:15:59,352 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/B is initiating minor compaction (all files) 2024-12-03T09:15:59,353 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/B in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:59,353 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/A in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:59,353 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/1ca8cb46cfc247e9af8b5e231d021acb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/3681826c492a4016876b0b467aef27b6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/ce7357a32bd842078b701df5c66fc582] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=39.7 K 2024-12-03T09:15:59,353 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/76f029183f8a4d4b8f824442040a760a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e9892308d2784422a3bfc10d805d8f92, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/3c09d7ed2e124e84a4e272d9951f8288] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=35.2 K 2024-12-03T09:15:59,353 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f029183f8a4d4b8f824442040a760a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733217356211 2024-12-03T09:15:59,353 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ca8cb46cfc247e9af8b5e231d021acb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733217356209 2024-12-03T09:15:59,354 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3681826c492a4016876b0b467aef27b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733217356265 2024-12-03T09:15:59,354 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting e9892308d2784422a3bfc10d805d8f92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733217356265 2024-12-03T09:15:59,355 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce7357a32bd842078b701df5c66fc582, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733217358410 2024-12-03T09:15:59,355 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c09d7ed2e124e84a4e272d9951f8288, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733217358411 2024-12-03T09:15:59,382 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#B#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:59,382 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#A#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:59,383 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/b736c05fd58f4b4db82e8585893ae180 is 50, key is test_row_0/B:col10/1733217358413/Put/seqid=0 2024-12-03T09:15:59,383 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/5d6afe50d2e2457f80aeebe378a96506 is 50, key is test_row_0/A:col10/1733217358413/Put/seqid=0 2024-12-03T09:15:59,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742240_1416 (size=12104) 2024-12-03T09:15:59,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742241_1417 (size=12104) 2024-12-03T09:15:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:15:59,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T09:15:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:15:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:15:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:15:59,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:15:59,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/b55ec9bcff394568b9f36ea2612daba3 is 50, key is test_row_0/A:col10/1733217358516/Put/seqid=0 2024-12-03T09:15:59,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742242_1418 (size=14341) 2024-12-03T09:15:59,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:59,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217419597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:59,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:59,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217419605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:59,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:59,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:59,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217419669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:59,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:59,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217419679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:59,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217419710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:59,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217419719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:59,811 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/5d6afe50d2e2457f80aeebe378a96506 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/5d6afe50d2e2457f80aeebe378a96506 2024-12-03T09:15:59,814 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/b736c05fd58f4b4db82e8585893ae180 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/b736c05fd58f4b4db82e8585893ae180 2024-12-03T09:15:59,816 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/A of 427f6f3f7c0d84ddc6f684fba93003f4 into 5d6afe50d2e2457f80aeebe378a96506(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:59,816 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:15:59,816 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/A, priority=13, startTime=1733217359348; duration=0sec 2024-12-03T09:15:59,816 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:15:59,816 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:A 2024-12-03T09:15:59,817 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:15:59,818 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:15:59,818 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/C is initiating minor compaction (all files) 2024-12-03T09:15:59,818 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/C in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:15:59,818 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/B of 427f6f3f7c0d84ddc6f684fba93003f4 into b736c05fd58f4b4db82e8585893ae180(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:15:59,818 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:15:59,818 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/4b1348939af449529c4eb76c682cf7f3, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/416004a2be904b83b38b4acb8cb84b92, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c085a8b9ba6b4b178d6a6579789bf416] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=35.2 K 2024-12-03T09:15:59,818 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/B, priority=13, startTime=1733217359348; duration=0sec 2024-12-03T09:15:59,818 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:15:59,818 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:B 2024-12-03T09:15:59,819 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b1348939af449529c4eb76c682cf7f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733217356211 2024-12-03T09:15:59,819 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 416004a2be904b83b38b4acb8cb84b92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733217356265 2024-12-03T09:15:59,819 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c085a8b9ba6b4b178d6a6579789bf416, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733217358411 2024-12-03T09:15:59,827 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#C#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:15:59,828 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/1a05b00dd39b4b1f87ff104e401526b4 is 50, key is test_row_0/C:col10/1733217358413/Put/seqid=0 2024-12-03T09:15:59,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742243_1419 (size=12104) 2024-12-03T09:15:59,879 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T09:15:59,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217419919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:59,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:15:59,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217419930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:15:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:15:59,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:15:59,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/b55ec9bcff394568b9f36ea2612daba3 2024-12-03T09:15:59,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/cde91bfc7c2348e184eaaf0b31106f7f is 50, key is test_row_0/B:col10/1733217358516/Put/seqid=0 2024-12-03T09:15:59,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742244_1420 (size=12001) 2024-12-03T09:16:00,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217420229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:00,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:00,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217420239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:00,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:00,250 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/1a05b00dd39b4b1f87ff104e401526b4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/1a05b00dd39b4b1f87ff104e401526b4 2024-12-03T09:16:00,254 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/C of 427f6f3f7c0d84ddc6f684fba93003f4 into 1a05b00dd39b4b1f87ff104e401526b4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:00,254 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:00,254 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/C, priority=13, startTime=1733217359349; duration=0sec 2024-12-03T09:16:00,255 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:00,255 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:C 2024-12-03T09:16:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T09:16:00,360 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:00,361 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-03T09:16:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T09:16:00,365 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:00,366 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:00,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:00,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/cde91bfc7c2348e184eaaf0b31106f7f 2024-12-03T09:16:00,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/68871811346341319e4e32fe25426031 is 50, key is test_row_0/C:col10/1733217358516/Put/seqid=0 2024-12-03T09:16:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217420440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:00,442 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 427f6f3f7c0d84ddc6f684fba93003f4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:00,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742245_1421 (size=12001) 2024-12-03T09:16:00,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/68871811346341319e4e32fe25426031 2024-12-03T09:16:00,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/b55ec9bcff394568b9f36ea2612daba3 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/b55ec9bcff394568b9f36ea2612daba3 2024-12-03T09:16:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T09:16:00,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/b55ec9bcff394568b9f36ea2612daba3, entries=200, sequenceid=80, filesize=14.0 K 2024-12-03T09:16:00,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/cde91bfc7c2348e184eaaf0b31106f7f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/cde91bfc7c2348e184eaaf0b31106f7f 2024-12-03T09:16:00,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/cde91bfc7c2348e184eaaf0b31106f7f, entries=150, sequenceid=80, filesize=11.7 K 2024-12-03T09:16:00,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/68871811346341319e4e32fe25426031 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/68871811346341319e4e32fe25426031 2024-12-03T09:16:00,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/68871811346341319e4e32fe25426031, entries=150, sequenceid=80, filesize=11.7 K 2024-12-03T09:16:00,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 427f6f3f7c0d84ddc6f684fba93003f4 in 955ms, sequenceid=80, compaction requested=false 2024-12-03T09:16:00,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:00,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T09:16:00,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:00,522 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T09:16:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:00,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/50bcae6267f74be384adbe22a5714856 is 50, key is test_row_0/A:col10/1733217359596/Put/seqid=0 2024-12-03T09:16:00,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742246_1422 (size=12001) 2024-12-03T09:16:00,600 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/50bcae6267f74be384adbe22a5714856 2024-12-03T09:16:00,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/9eb529a09acf4806828a53a6816758f4 is 50, key is test_row_0/B:col10/1733217359596/Put/seqid=0 2024-12-03T09:16:00,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742247_1423 (size=12001) 2024-12-03T09:16:00,620 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/9eb529a09acf4806828a53a6816758f4 2024-12-03T09:16:00,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/b40ee0c06ad34f629692712a3363443a is 50, key is test_row_0/C:col10/1733217359596/Put/seqid=0 2024-12-03T09:16:00,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742248_1424 (size=12001) 2024-12-03T09:16:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T09:16:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:00,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:00,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217420746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:00,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217420749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:00,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217420750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:00,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217420751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:00,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217420859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217420870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217420870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:00,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T09:16:01,054 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/b40ee0c06ad34f629692712a3363443a 2024-12-03T09:16:01,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/50bcae6267f74be384adbe22a5714856 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/50bcae6267f74be384adbe22a5714856 2024-12-03T09:16:01,073 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/50bcae6267f74be384adbe22a5714856, entries=150, sequenceid=95, filesize=11.7 K 2024-12-03T09:16:01,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/9eb529a09acf4806828a53a6816758f4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/9eb529a09acf4806828a53a6816758f4 2024-12-03T09:16:01,080 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/9eb529a09acf4806828a53a6816758f4, entries=150, sequenceid=95, filesize=11.7 K 2024-12-03T09:16:01,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/b40ee0c06ad34f629692712a3363443a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/b40ee0c06ad34f629692712a3363443a 2024-12-03T09:16:01,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217421069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217421080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:01,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217421080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:01,088 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/b40ee0c06ad34f629692712a3363443a, entries=150, sequenceid=95, filesize=11.7 K 2024-12-03T09:16:01,089 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 427f6f3f7c0d84ddc6f684fba93003f4 in 568ms, sequenceid=95, compaction requested=true 2024-12-03T09:16:01,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:01,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:01,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-03T09:16:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-03T09:16:01,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-03T09:16:01,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 724 msec 2024-12-03T09:16:01,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 737 msec 2024-12-03T09:16:01,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217360728, max timestamp=1733217361397Num uniques 859; 2024-12-03T09:16:01,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:01,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-12-03T09:16:01,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217360728, max timestamp=1733217361397Num uniques 859; 2024-12-03T09:16:01,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217360728, max timestamp=1733217361397Num uniques 859; 2024-12-03T09:16:01,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:01,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:01,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:01,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:01,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:01,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:01,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:01,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:01,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/a16df14416e04e2f91d0905b802436fe is 50, key is test_row_0/A:col10/1733217361400/Put/seqid=0 2024-12-03T09:16:01,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742249_1425 (size=16681) 2024-12-03T09:16:01,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733217361440, max timestamp=1733217361479Num uniques 859; 2024-12-03T09:16:01,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T09:16:01,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733217361440, max timestamp=1733217361479Num uniques 859; 2024-12-03T09:16:01,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T09:16:01,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733217361440, max timestamp=1733217361479Num uniques 859; 2024-12-03T09:16:01,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T09:16:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T09:16:01,499 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:01,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-03T09:16:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T09:16:01,501 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:01,501 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:01,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:01,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217421554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:01,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217421570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:01,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217421571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T09:16:01,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T09:16:01,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:01,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:01,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:01,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:01,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:01,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217421679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:01,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:01,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217421689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:01,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217421690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:01,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:01,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217421769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:01,774 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 427f6f3f7c0d84ddc6f684fba93003f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:01,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T09:16:01,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:01,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:01,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:01,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:01,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:01,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T09:16:01,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/a16df14416e04e2f91d0905b802436fe 2024-12-03T09:16:01,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/243661c488da4ff5bd1a039682fbe4b7 is 50, key is test_row_0/B:col10/1733217361400/Put/seqid=0 2024-12-03T09:16:01,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742250_1426 (size=12001) 2024-12-03T09:16:01,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217421899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:01,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:01,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217421910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217421910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:01,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T09:16:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:01,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:02,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T09:16:02,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:02,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:02,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:02,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:02,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T09:16:02,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217422209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:02,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217422219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:02,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:02,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217422230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:02,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/243661c488da4ff5bd1a039682fbe4b7 2024-12-03T09:16:02,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/b25d525f64a4439d9f122d5ad5944f0d is 50, key is test_row_0/C:col10/1733217361400/Put/seqid=0 2024-12-03T09:16:02,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742251_1427 (size=12001) 2024-12-03T09:16:02,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T09:16:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:02,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:02,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T09:16:02,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:02,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:02,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:02,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:02,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:02,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T09:16:02,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:02,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:02,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:02,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:02,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T09:16:02,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/b25d525f64a4439d9f122d5ad5944f0d 2024-12-03T09:16:02,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/a16df14416e04e2f91d0905b802436fe as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/a16df14416e04e2f91d0905b802436fe 2024-12-03T09:16:02,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/a16df14416e04e2f91d0905b802436fe, entries=250, sequenceid=118, filesize=16.3 K 2024-12-03T09:16:02,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/243661c488da4ff5bd1a039682fbe4b7 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/243661c488da4ff5bd1a039682fbe4b7 2024-12-03T09:16:02,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/243661c488da4ff5bd1a039682fbe4b7, entries=150, sequenceid=118, filesize=11.7 K 2024-12-03T09:16:02,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/b25d525f64a4439d9f122d5ad5944f0d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/b25d525f64a4439d9f122d5ad5944f0d 2024-12-03T09:16:02,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/b25d525f64a4439d9f122d5ad5944f0d, entries=150, sequenceid=118, filesize=11.7 K 2024-12-03T09:16:02,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=147.60 KB/151140 for 427f6f3f7c0d84ddc6f684fba93003f4 in 1275ms, sequenceid=118, compaction requested=true 2024-12-03T09:16:02,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:02,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:02,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:02,679 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:02,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:02,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:02,679 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:02,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:02,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:02,680 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55127 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:02,680 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:02,680 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/B is initiating minor compaction (all files) 2024-12-03T09:16:02,680 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/A is initiating minor compaction (all files) 2024-12-03T09:16:02,680 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/B in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:02,680 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/A in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:02,680 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/5d6afe50d2e2457f80aeebe378a96506, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/b55ec9bcff394568b9f36ea2612daba3, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/50bcae6267f74be384adbe22a5714856, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/a16df14416e04e2f91d0905b802436fe] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=53.8 K 2024-12-03T09:16:02,680 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/b736c05fd58f4b4db82e8585893ae180, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/cde91bfc7c2348e184eaaf0b31106f7f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/9eb529a09acf4806828a53a6816758f4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/243661c488da4ff5bd1a039682fbe4b7] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=47.0 K 2024-12-03T09:16:02,680 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting b736c05fd58f4b4db82e8585893ae180, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733217358411 2024-12-03T09:16:02,680 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d6afe50d2e2457f80aeebe378a96506, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733217358411 2024-12-03T09:16:02,681 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting cde91bfc7c2348e184eaaf0b31106f7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733217358516 2024-12-03T09:16:02,681 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b55ec9bcff394568b9f36ea2612daba3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733217358516 2024-12-03T09:16:02,681 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50bcae6267f74be384adbe22a5714856, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733217359580 2024-12-03T09:16:02,682 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eb529a09acf4806828a53a6816758f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733217359580 2024-12-03T09:16:02,683 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a16df14416e04e2f91d0905b802436fe, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733217360728 2024-12-03T09:16:02,683 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 243661c488da4ff5bd1a039682fbe4b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733217361397 2024-12-03T09:16:02,691 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:02,692 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/1e2ca974b2e548d89a6e2a171618a7b8 is 50, key is test_row_0/B:col10/1733217361400/Put/seqid=0 2024-12-03T09:16:02,693 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#A#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:02,694 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/a4fc4a0349dd4d029c9b004fc027e870 is 50, key is test_row_0/A:col10/1733217361400/Put/seqid=0 2024-12-03T09:16:02,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742252_1428 (size=12241) 2024-12-03T09:16:02,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742253_1429 (size=12241) 2024-12-03T09:16:02,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T09:16:02,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:02,723 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-03T09:16:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:02,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/7d5752d999034937843e6c8aafffb1b9 is 50, key is test_row_0/A:col10/1733217361568/Put/seqid=0 2024-12-03T09:16:02,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742254_1430 (size=12151) 2024-12-03T09:16:02,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:02,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:02,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217422749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:02,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217422760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217422761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:02,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217422870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:02,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217422879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:02,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217422879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217423080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217423099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217423099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,107 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/1e2ca974b2e548d89a6e2a171618a7b8 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/1e2ca974b2e548d89a6e2a171618a7b8 2024-12-03T09:16:03,107 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/a4fc4a0349dd4d029c9b004fc027e870 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/a4fc4a0349dd4d029c9b004fc027e870 2024-12-03T09:16:03,120 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/A of 427f6f3f7c0d84ddc6f684fba93003f4 into a4fc4a0349dd4d029c9b004fc027e870(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:03,120 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:03,120 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/A, priority=12, startTime=1733217362679; duration=0sec 2024-12-03T09:16:03,120 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:03,120 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:A 2024-12-03T09:16:03,121 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:03,121 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/B of 427f6f3f7c0d84ddc6f684fba93003f4 into 1e2ca974b2e548d89a6e2a171618a7b8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:03,121 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:03,121 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/B, priority=12, startTime=1733217362679; duration=0sec 2024-12-03T09:16:03,121 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:03,121 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:B 2024-12-03T09:16:03,123 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:03,123 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/C is initiating minor compaction (all files) 2024-12-03T09:16:03,124 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/C in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:03,124 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/1a05b00dd39b4b1f87ff104e401526b4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/68871811346341319e4e32fe25426031, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/b40ee0c06ad34f629692712a3363443a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/b25d525f64a4439d9f122d5ad5944f0d] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=47.0 K 2024-12-03T09:16:03,124 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a05b00dd39b4b1f87ff104e401526b4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733217358411 2024-12-03T09:16:03,125 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68871811346341319e4e32fe25426031, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1733217358516 2024-12-03T09:16:03,125 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40ee0c06ad34f629692712a3363443a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733217359580 2024-12-03T09:16:03,125 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b25d525f64a4439d9f122d5ad5944f0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733217361397 2024-12-03T09:16:03,131 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/7d5752d999034937843e6c8aafffb1b9 2024-12-03T09:16:03,136 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#C#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:03,136 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/39ae1beba8cc4456ae841b1be31c4c23 is 50, key is test_row_0/C:col10/1733217361400/Put/seqid=0 2024-12-03T09:16:03,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/3b0d5b9616084571a84e445d42d49574 is 50, key is test_row_0/B:col10/1733217361568/Put/seqid=0 2024-12-03T09:16:03,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742255_1431 (size=12241) 2024-12-03T09:16:03,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742256_1432 (size=12151) 2024-12-03T09:16:03,155 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/3b0d5b9616084571a84e445d42d49574 2024-12-03T09:16:03,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/f82163086e8941aeb3a797c0a4475ec4 is 50, key is test_row_0/C:col10/1733217361568/Put/seqid=0 2024-12-03T09:16:03,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742257_1433 (size=12151) 2024-12-03T09:16:03,170 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/f82163086e8941aeb3a797c0a4475ec4 2024-12-03T09:16:03,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/7d5752d999034937843e6c8aafffb1b9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/7d5752d999034937843e6c8aafffb1b9 2024-12-03T09:16:03,189 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/7d5752d999034937843e6c8aafffb1b9, entries=150, sequenceid=155, filesize=11.9 K 2024-12-03T09:16:03,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/3b0d5b9616084571a84e445d42d49574 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/3b0d5b9616084571a84e445d42d49574 2024-12-03T09:16:03,197 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/3b0d5b9616084571a84e445d42d49574, entries=150, sequenceid=155, filesize=11.9 K 2024-12-03T09:16:03,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/f82163086e8941aeb3a797c0a4475ec4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/f82163086e8941aeb3a797c0a4475ec4 2024-12-03T09:16:03,205 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/f82163086e8941aeb3a797c0a4475ec4, entries=150, sequenceid=155, filesize=11.9 K 2024-12-03T09:16:03,206 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 427f6f3f7c0d84ddc6f684fba93003f4 in 483ms, sequenceid=155, compaction requested=false 2024-12-03T09:16:03,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:03,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:03,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-03T09:16:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-03T09:16:03,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-03T09:16:03,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7080 sec 2024-12-03T09:16:03,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.7110 sec 2024-12-03T09:16:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:03,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:16:03,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:03,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:03,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:03,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:03,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:03,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:03,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/6b0c3f87c83d4370acef4adbf6ffa240 is 50, key is test_row_0/A:col10/1733217362759/Put/seqid=0 2024-12-03T09:16:03,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742258_1434 (size=14541) 2024-12-03T09:16:03,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217363436, max timestamp=1733217363447Num uniques 859; 2024-12-03T09:16:03,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:16:03,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217363436, max timestamp=1733217363447Num uniques 859; 2024-12-03T09:16:03,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:16:03,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217363436, max timestamp=1733217363447Num uniques 859; 2024-12-03T09:16:03,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:16:03,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217423483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217423511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:03,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217423511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,549 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/39ae1beba8cc4456ae841b1be31c4c23 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/39ae1beba8cc4456ae841b1be31c4c23 2024-12-03T09:16:03,553 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/C of 427f6f3f7c0d84ddc6f684fba93003f4 into 39ae1beba8cc4456ae841b1be31c4c23(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:03,553 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:03,553 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/C, priority=12, startTime=1733217362679; duration=0sec 2024-12-03T09:16:03,553 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:03,553 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:C 2024-12-03T09:16:03,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217423619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:03,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217423640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217423640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T09:16:03,649 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:03,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-03T09:16:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T09:16:03,652 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:03,652 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:03,653 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T09:16:03,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217423789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:03,797 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 427f6f3f7c0d84ddc6f684fba93003f4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/6b0c3f87c83d4370acef4adbf6ffa240 2024-12-03T09:16:03,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T09:16:03,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:03,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:03,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:03,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/92f1a711295c4e0386e12376570ad12a is 50, key is test_row_0/B:col10/1733217362759/Put/seqid=0 2024-12-03T09:16:03,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742259_1435 (size=12151) 2024-12-03T09:16:03,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/92f1a711295c4e0386e12376570ad12a 2024-12-03T09:16:03,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/268fd085de7940c8831342623bd0e684 is 50, key is test_row_0/C:col10/1733217362759/Put/seqid=0 2024-12-03T09:16:03,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742260_1436 (size=12151) 2024-12-03T09:16:03,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217423829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:03,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217423849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217423849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:03,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T09:16:03,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:03,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:03,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:03,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:03,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:03,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T09:16:04,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T09:16:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:04,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:04,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217424139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217424169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:04,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217424169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:04,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/268fd085de7940c8831342623bd0e684 2024-12-03T09:16:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/6b0c3f87c83d4370acef4adbf6ffa240 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6b0c3f87c83d4370acef4adbf6ffa240 2024-12-03T09:16:04,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6b0c3f87c83d4370acef4adbf6ffa240, entries=200, sequenceid=167, filesize=14.2 K 2024-12-03T09:16:04,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/92f1a711295c4e0386e12376570ad12a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/92f1a711295c4e0386e12376570ad12a 2024-12-03T09:16:04,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/92f1a711295c4e0386e12376570ad12a, entries=150, sequenceid=167, filesize=11.9 K 2024-12-03T09:16:04,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/268fd085de7940c8831342623bd0e684 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/268fd085de7940c8831342623bd0e684 2024-12-03T09:16:04,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/268fd085de7940c8831342623bd0e684, entries=150, sequenceid=167, filesize=11.9 K 2024-12-03T09:16:04,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 427f6f3f7c0d84ddc6f684fba93003f4 in 852ms, sequenceid=167, compaction requested=true 2024-12-03T09:16:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:04,247 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:04,247 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:04,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:04,248 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:04,248 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:04,248 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/A is initiating minor compaction (all files) 2024-12-03T09:16:04,248 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/B is initiating minor compaction (all files) 2024-12-03T09:16:04,248 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/A in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:04,248 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/B in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:04,248 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/a4fc4a0349dd4d029c9b004fc027e870, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/7d5752d999034937843e6c8aafffb1b9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6b0c3f87c83d4370acef4adbf6ffa240] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=38.0 K 2024-12-03T09:16:04,248 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/1e2ca974b2e548d89a6e2a171618a7b8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/3b0d5b9616084571a84e445d42d49574, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/92f1a711295c4e0386e12376570ad12a] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=35.7 K 2024-12-03T09:16:04,248 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e2ca974b2e548d89a6e2a171618a7b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733217361397 2024-12-03T09:16:04,249 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4fc4a0349dd4d029c9b004fc027e870, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733217361397 2024-12-03T09:16:04,249 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b0d5b9616084571a84e445d42d49574, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733217361531 2024-12-03T09:16:04,249 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d5752d999034937843e6c8aafffb1b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733217361531 2024-12-03T09:16:04,249 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f1a711295c4e0386e12376570ad12a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733217362759 2024-12-03T09:16:04,249 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b0c3f87c83d4370acef4adbf6ffa240, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733217362734 2024-12-03T09:16:04,257 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#A#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:04,257 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/6b3e9dc2373c49a7945b91d35d884897 is 50, key is test_row_0/A:col10/1733217362759/Put/seqid=0 2024-12-03T09:16:04,257 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#B#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:04,258 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/09efcb2cbe3148c6aae06938aa8bf495 is 50, key is test_row_0/B:col10/1733217362759/Put/seqid=0 2024-12-03T09:16:04,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T09:16:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:04,265 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-03T09:16:04,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742262_1438 (size=12493) 2024-12-03T09:16:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:04,270 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/09efcb2cbe3148c6aae06938aa8bf495 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/09efcb2cbe3148c6aae06938aa8bf495 2024-12-03T09:16:04,274 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/B of 427f6f3f7c0d84ddc6f684fba93003f4 into 09efcb2cbe3148c6aae06938aa8bf495(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:04,274 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:04,274 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/B, priority=13, startTime=1733217364247; duration=0sec 2024-12-03T09:16:04,274 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:04,274 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:B 2024-12-03T09:16:04,274 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:04,275 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:04,275 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/C is initiating minor compaction (all files) 2024-12-03T09:16:04,275 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/C in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:04,276 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/39ae1beba8cc4456ae841b1be31c4c23, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/f82163086e8941aeb3a797c0a4475ec4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/268fd085de7940c8831342623bd0e684] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=35.7 K 2024-12-03T09:16:04,276 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 39ae1beba8cc4456ae841b1be31c4c23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733217361397 2024-12-03T09:16:04,276 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f82163086e8941aeb3a797c0a4475ec4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733217361531 2024-12-03T09:16:04,276 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 268fd085de7940c8831342623bd0e684, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733217362759 2024-12-03T09:16:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T09:16:04,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/b28f0a22eff04779bf380ce6cfcf9904 is 50, key is test_row_0/A:col10/1733217363484/Put/seqid=0 2024-12-03T09:16:04,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742261_1437 (size=12493) 2024-12-03T09:16:04,290 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#C#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:04,291 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/824a3394c4a2460b897f9a3ca00a05ab is 50, key is test_row_0/C:col10/1733217362759/Put/seqid=0 2024-12-03T09:16:04,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742263_1439 (size=12151) 2024-12-03T09:16:04,296 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/b28f0a22eff04779bf380ce6cfcf9904 2024-12-03T09:16:04,297 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/6b3e9dc2373c49a7945b91d35d884897 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6b3e9dc2373c49a7945b91d35d884897 2024-12-03T09:16:04,305 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/A of 427f6f3f7c0d84ddc6f684fba93003f4 into 6b3e9dc2373c49a7945b91d35d884897(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:04,305 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:04,305 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/A, priority=13, startTime=1733217364247; duration=0sec 2024-12-03T09:16:04,305 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:04,305 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:A 2024-12-03T09:16:04,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742264_1440 (size=12493) 2024-12-03T09:16:04,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/1a55c23d25f544e8a9c64cf57e4fca41 is 50, key is test_row_0/B:col10/1733217363484/Put/seqid=0 2024-12-03T09:16:04,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742265_1441 (size=12151) 2024-12-03T09:16:04,319 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/824a3394c4a2460b897f9a3ca00a05ab as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/824a3394c4a2460b897f9a3ca00a05ab 2024-12-03T09:16:04,320 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/1a55c23d25f544e8a9c64cf57e4fca41 2024-12-03T09:16:04,325 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/C of 427f6f3f7c0d84ddc6f684fba93003f4 into 824a3394c4a2460b897f9a3ca00a05ab(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:04,325 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:04,325 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/C, priority=13, startTime=1733217364247; duration=0sec 2024-12-03T09:16:04,325 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:04,325 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:C 2024-12-03T09:16:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/76018d3b5ded4a64988897355fb2e234 is 50, key is test_row_0/C:col10/1733217363484/Put/seqid=0 2024-12-03T09:16:04,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742266_1442 (size=12151) 2024-12-03T09:16:04,356 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/76018d3b5ded4a64988897355fb2e234 2024-12-03T09:16:04,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/b28f0a22eff04779bf380ce6cfcf9904 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/b28f0a22eff04779bf380ce6cfcf9904 2024-12-03T09:16:04,364 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/b28f0a22eff04779bf380ce6cfcf9904, entries=150, sequenceid=204, filesize=11.9 K 2024-12-03T09:16:04,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/1a55c23d25f544e8a9c64cf57e4fca41 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/1a55c23d25f544e8a9c64cf57e4fca41 2024-12-03T09:16:04,369 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/1a55c23d25f544e8a9c64cf57e4fca41, entries=150, sequenceid=204, filesize=11.9 K 2024-12-03T09:16:04,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/76018d3b5ded4a64988897355fb2e234 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/76018d3b5ded4a64988897355fb2e234 2024-12-03T09:16:04,373 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/76018d3b5ded4a64988897355fb2e234, entries=150, sequenceid=204, filesize=11.9 K 2024-12-03T09:16:04,375 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=0 B/0 for 427f6f3f7c0d84ddc6f684fba93003f4 in 110ms, sequenceid=204, compaction requested=false 2024-12-03T09:16:04,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:04,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:04,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-03T09:16:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-03T09:16:04,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-03T09:16:04,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 723 msec 2024-12-03T09:16:04,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 730 msec 2024-12-03T09:16:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:04,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:16:04,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:04,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:04,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:04,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:04,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:04,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:04,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/d029ad9d90704553ae7dec5cfbf1054b is 50, key is test_row_1/A:col10/1733217364483/Put/seqid=0 2024-12-03T09:16:04,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742267_1443 (size=9755) 2024-12-03T09:16:04,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217364536, max timestamp=1733217364564Num uniques 859; 2024-12-03T09:16:04,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:16:04,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217364536, max timestamp=1733217364564Num uniques 859; 2024-12-03T09:16:04,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:16:04,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217364536, max timestamp=1733217364564Num uniques 859; 2024-12-03T09:16:04,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:16:04,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217424649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:04,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217424659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:04,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217424679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217424679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:04,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:04,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217424769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:04,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:04,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T09:16:04,789 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:04,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:04,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-03T09:16:04,791 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:04,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T09:16:04,792 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:04,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:04,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/d029ad9d90704553ae7dec5cfbf1054b 2024-12-03T09:16:04,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/7da0bf9c46394df8bf1af01addb829eb is 50, key is test_row_1/B:col10/1733217364483/Put/seqid=0 2024-12-03T09:16:04,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T09:16:04,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742268_1444 (size=7365) 2024-12-03T09:16:04,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T09:16:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:04,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:04,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:04,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:04,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:04,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:04,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217424980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:05,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T09:16:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T09:16:05,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T09:16:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217425289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:05,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:05,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:05,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:05,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/7da0bf9c46394df8bf1af01addb829eb 2024-12-03T09:16:05,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/81eb98f276bf4fd094122d2e6474f1eb is 50, key is test_row_1/C:col10/1733217364483/Put/seqid=0 2024-12-03T09:16:05,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742269_1445 (size=7365) 2024-12-03T09:16:05,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T09:16:05,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:05,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T09:16:05,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T09:16:05,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:05,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217425669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:05,674 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 427f6f3f7c0d84ddc6f684fba93003f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:05,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217425700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:05,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217425700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:05,705 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 427f6f3f7c0d84ddc6f684fba93003f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:05,706 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 427f6f3f7c0d84ddc6f684fba93003f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:05,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T09:16:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/81eb98f276bf4fd094122d2e6474f1eb 2024-12-03T09:16:05,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/d029ad9d90704553ae7dec5cfbf1054b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/d029ad9d90704553ae7dec5cfbf1054b 2024-12-03T09:16:05,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/d029ad9d90704553ae7dec5cfbf1054b, entries=100, sequenceid=217, filesize=9.5 K 2024-12-03T09:16:05,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/7da0bf9c46394df8bf1af01addb829eb as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/7da0bf9c46394df8bf1af01addb829eb 2024-12-03T09:16:05,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/7da0bf9c46394df8bf1af01addb829eb, entries=50, sequenceid=217, filesize=7.2 K 2024-12-03T09:16:05,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/81eb98f276bf4fd094122d2e6474f1eb as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/81eb98f276bf4fd094122d2e6474f1eb 2024-12-03T09:16:05,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/81eb98f276bf4fd094122d2e6474f1eb, entries=50, sequenceid=217, filesize=7.2 K 2024-12-03T09:16:05,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 427f6f3f7c0d84ddc6f684fba93003f4 in 1248ms, sequenceid=217, compaction requested=true 2024-12-03T09:16:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:05,732 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:05,732 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:05,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:05,733 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:05,733 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:05,733 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/B is initiating minor compaction (all files) 2024-12-03T09:16:05,733 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/A is initiating minor compaction (all files) 2024-12-03T09:16:05,733 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/B in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,733 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/A in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,733 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/09efcb2cbe3148c6aae06938aa8bf495, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/1a55c23d25f544e8a9c64cf57e4fca41, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/7da0bf9c46394df8bf1af01addb829eb] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=31.3 K 2024-12-03T09:16:05,733 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6b3e9dc2373c49a7945b91d35d884897, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/b28f0a22eff04779bf380ce6cfcf9904, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/d029ad9d90704553ae7dec5cfbf1054b] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=33.6 K 2024-12-03T09:16:05,733 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 09efcb2cbe3148c6aae06938aa8bf495, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733217362759 2024-12-03T09:16:05,733 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b3e9dc2373c49a7945b91d35d884897, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733217362759 2024-12-03T09:16:05,733 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a55c23d25f544e8a9c64cf57e4fca41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733217363470 2024-12-03T09:16:05,733 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b28f0a22eff04779bf380ce6cfcf9904, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733217363470 2024-12-03T09:16:05,734 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7da0bf9c46394df8bf1af01addb829eb, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733217364483 2024-12-03T09:16:05,734 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d029ad9d90704553ae7dec5cfbf1054b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733217364480 2024-12-03T09:16:05,739 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#B#compaction#372 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:05,740 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/68d6fb9d7dbf4d349f6476be012f5496 is 50, key is test_row_0/B:col10/1733217363484/Put/seqid=0 2024-12-03T09:16:05,740 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#A#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:05,740 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/cfc1e512f3e04162815879bf5f135cb9 is 50, key is test_row_0/A:col10/1733217363484/Put/seqid=0 2024-12-03T09:16:05,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742270_1446 (size=12595) 2024-12-03T09:16:05,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742271_1447 (size=12595) 2024-12-03T09:16:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:05,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T09:16:05,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:05,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:05,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:05,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:05,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:05,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:05,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/55f1ebf18ea04868ab267cb09fd5afaf is 50, key is test_row_0/A:col10/1733217364621/Put/seqid=0 2024-12-03T09:16:05,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742272_1448 (size=14541) 2024-12-03T09:16:05,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/55f1ebf18ea04868ab267cb09fd5afaf 2024-12-03T09:16:05,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/b566e1161da242bf96b8fb55d3b55d19 is 50, key is test_row_0/B:col10/1733217364621/Put/seqid=0 2024-12-03T09:16:05,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742273_1449 (size=12151) 2024-12-03T09:16:05,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/b566e1161da242bf96b8fb55d3b55d19 2024-12-03T09:16:05,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/055c1ed5f27a4dce8a6f429b48a73e59 is 50, key is test_row_0/C:col10/1733217364621/Put/seqid=0 2024-12-03T09:16:05,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742274_1450 (size=12151) 2024-12-03T09:16:05,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/055c1ed5f27a4dce8a6f429b48a73e59 2024-12-03T09:16:05,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/55f1ebf18ea04868ab267cb09fd5afaf as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/55f1ebf18ea04868ab267cb09fd5afaf 2024-12-03T09:16:05,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/55f1ebf18ea04868ab267cb09fd5afaf, entries=200, sequenceid=254, filesize=14.2 K 2024-12-03T09:16:05,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/b566e1161da242bf96b8fb55d3b55d19 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/b566e1161da242bf96b8fb55d3b55d19 2024-12-03T09:16:05,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/b566e1161da242bf96b8fb55d3b55d19, entries=150, sequenceid=254, filesize=11.9 K 2024-12-03T09:16:05,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/055c1ed5f27a4dce8a6f429b48a73e59 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/055c1ed5f27a4dce8a6f429b48a73e59 2024-12-03T09:16:05,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/055c1ed5f27a4dce8a6f429b48a73e59, entries=150, sequenceid=254, filesize=11.9 K 2024-12-03T09:16:05,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 427f6f3f7c0d84ddc6f684fba93003f4 in 49ms, sequenceid=254, compaction requested=true 2024-12-03T09:16:05,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:05,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:A, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:05,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T09:16:05,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:B, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:05,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T09:16:05,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:05,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T09:16:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:05,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:16:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:05,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T09:16:05,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:05,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:05,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:05,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/809076f9f9f849749826d1b5ee8cf135 is 50, key is test_row_0/A:col10/1733217365836/Put/seqid=0 2024-12-03T09:16:05,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742275_1451 (size=14741) 2024-12-03T09:16:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T09:16:06,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217365943, max timestamp=1733217366006Num uniques 859; 2024-12-03T09:16:06,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:06,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217365943, max timestamp=1733217366006Num uniques 859; 2024-12-03T09:16:06,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:06,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217365943, max timestamp=1733217366006Num uniques 859; 2024-12-03T09:16:06,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:06,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T09:16:06,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:06,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:06,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217426144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:06,153 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/68d6fb9d7dbf4d349f6476be012f5496 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/68d6fb9d7dbf4d349f6476be012f5496 2024-12-03T09:16:06,158 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/B of 427f6f3f7c0d84ddc6f684fba93003f4 into 68d6fb9d7dbf4d349f6476be012f5496(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:06,158 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:06,158 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/B, priority=13, startTime=1733217365732; duration=0sec 2024-12-03T09:16:06,158 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/cfc1e512f3e04162815879bf5f135cb9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/cfc1e512f3e04162815879bf5f135cb9 2024-12-03T09:16:06,158 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T09:16:06,158 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:B 2024-12-03T09:16:06,158 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-12-03T09:16:06,158 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T09:16:06,158 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T09:16:06,158 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. because compaction request was cancelled 2024-12-03T09:16:06,158 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:A 2024-12-03T09:16:06,158 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:06,159 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44160 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:06,159 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/C is initiating minor compaction (all files) 2024-12-03T09:16:06,159 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/C in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,159 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/824a3394c4a2460b897f9a3ca00a05ab, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/76018d3b5ded4a64988897355fb2e234, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/81eb98f276bf4fd094122d2e6474f1eb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/055c1ed5f27a4dce8a6f429b48a73e59] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=43.1 K 2024-12-03T09:16:06,160 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 824a3394c4a2460b897f9a3ca00a05ab, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733217362759 2024-12-03T09:16:06,160 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 76018d3b5ded4a64988897355fb2e234, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733217363470 2024-12-03T09:16:06,160 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 81eb98f276bf4fd094122d2e6474f1eb, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733217364483 2024-12-03T09:16:06,160 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 055c1ed5f27a4dce8a6f429b48a73e59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733217364621 2024-12-03T09:16:06,162 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/A of 427f6f3f7c0d84ddc6f684fba93003f4 into cfc1e512f3e04162815879bf5f135cb9(size=12.3 K), total size for store is 26.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:06,162 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:06,162 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/A, priority=13, startTime=1733217365732; duration=0sec 2024-12-03T09:16:06,162 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T09:16:06,162 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:A 2024-12-03T09:16:06,162 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:B 2024-12-03T09:16:06,162 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-12-03T09:16:06,162 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T09:16:06,162 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T09:16:06,162 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. because compaction request was cancelled 2024-12-03T09:16:06,162 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:C 2024-12-03T09:16:06,162 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T09:16:06,163 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T09:16:06,163 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T09:16:06,163 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. because compaction request was cancelled 2024-12-03T09:16:06,163 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:B 2024-12-03T09:16:06,166 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#C#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:06,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T09:16:06,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:06,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,167 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c4b6a51e5f5c4d1ca61241a4f1cba64b is 50, key is test_row_0/C:col10/1733217364621/Put/seqid=0 2024-12-03T09:16:06,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742276_1452 (size=12629) 2024-12-03T09:16:06,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217426259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:06,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:06,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/809076f9f9f849749826d1b5ee8cf135 2024-12-03T09:16:06,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/e41bf757d714421a9dfd239c46213d65 is 50, key is test_row_0/B:col10/1733217365836/Put/seqid=0 2024-12-03T09:16:06,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742277_1453 (size=12301) 2024-12-03T09:16:06,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/e41bf757d714421a9dfd239c46213d65 2024-12-03T09:16:06,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/67ee1264d9b04374a5ad4b7635034bed is 50, key is test_row_0/C:col10/1733217365836/Put/seqid=0 2024-12-03T09:16:06,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742278_1454 (size=12301) 2024-12-03T09:16:06,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T09:16:06,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:06,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T09:16:06,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:06,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217426469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:06,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:06,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:06,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:06,575 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c4b6a51e5f5c4d1ca61241a4f1cba64b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c4b6a51e5f5c4d1ca61241a4f1cba64b 2024-12-03T09:16:06,578 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/C of 427f6f3f7c0d84ddc6f684fba93003f4 into c4b6a51e5f5c4d1ca61241a4f1cba64b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:06,578 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:06,578 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/C, priority=12, startTime=1733217365854; duration=0sec 2024-12-03T09:16:06,578 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:06,578 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:C 2024-12-03T09:16:06,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T09:16:06,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:06,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:06,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/67ee1264d9b04374a5ad4b7635034bed 2024-12-03T09:16:06,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/809076f9f9f849749826d1b5ee8cf135 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/809076f9f9f849749826d1b5ee8cf135 2024-12-03T09:16:06,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/809076f9f9f849749826d1b5ee8cf135, entries=200, sequenceid=264, filesize=14.4 K 2024-12-03T09:16:06,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/e41bf757d714421a9dfd239c46213d65 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e41bf757d714421a9dfd239c46213d65 2024-12-03T09:16:06,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e41bf757d714421a9dfd239c46213d65, entries=150, sequenceid=264, filesize=12.0 K 2024-12-03T09:16:06,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/67ee1264d9b04374a5ad4b7635034bed as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/67ee1264d9b04374a5ad4b7635034bed 2024-12-03T09:16:06,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/67ee1264d9b04374a5ad4b7635034bed, entries=150, sequenceid=264, filesize=12.0 K 2024-12-03T09:16:06,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 427f6f3f7c0d84ddc6f684fba93003f4 in 843ms, sequenceid=264, compaction requested=true 2024-12-03T09:16:06,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:06,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:06,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:06,703 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:06,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:06,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:06,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:06,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T09:16:06,704 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41877 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:06,704 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/A is initiating minor compaction (all files) 2024-12-03T09:16:06,704 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/A in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,704 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/cfc1e512f3e04162815879bf5f135cb9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/55f1ebf18ea04868ab267cb09fd5afaf, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/809076f9f9f849749826d1b5ee8cf135] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=40.9 K 2024-12-03T09:16:06,704 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:06,705 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfc1e512f3e04162815879bf5f135cb9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733217363470 2024-12-03T09:16:06,705 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55f1ebf18ea04868ab267cb09fd5afaf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733217364621 2024-12-03T09:16:06,705 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 809076f9f9f849749826d1b5ee8cf135, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733217365836 2024-12-03T09:16:06,705 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:06,705 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/B is initiating minor compaction (all files) 2024-12-03T09:16:06,705 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/B in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,705 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/68d6fb9d7dbf4d349f6476be012f5496, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/b566e1161da242bf96b8fb55d3b55d19, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e41bf757d714421a9dfd239c46213d65] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=36.2 K 2024-12-03T09:16:06,706 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 68d6fb9d7dbf4d349f6476be012f5496, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733217363470 2024-12-03T09:16:06,706 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting b566e1161da242bf96b8fb55d3b55d19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733217364621 2024-12-03T09:16:06,706 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting e41bf757d714421a9dfd239c46213d65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733217365836 2024-12-03T09:16:06,712 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#A#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:06,712 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/668efbdb0db3492ba31e9cb290f3b25f is 50, key is test_row_0/A:col10/1733217365836/Put/seqid=0 2024-12-03T09:16:06,714 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#B#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:06,714 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/7be56f38947d418ab29850d91611fea0 is 50, key is test_row_0/B:col10/1733217365836/Put/seqid=0 2024-12-03T09:16:06,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742280_1456 (size=12847) 2024-12-03T09:16:06,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742279_1455 (size=12847) 2024-12-03T09:16:06,721 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/7be56f38947d418ab29850d91611fea0 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/7be56f38947d418ab29850d91611fea0 2024-12-03T09:16:06,725 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/B of 427f6f3f7c0d84ddc6f684fba93003f4 into 7be56f38947d418ab29850d91611fea0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:06,725 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:06,725 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/B, priority=13, startTime=1733217366703; duration=0sec 2024-12-03T09:16:06,725 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:06,725 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:B 2024-12-03T09:16:06,725 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T09:16:06,725 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T09:16:06,726 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T09:16:06,726 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. because compaction request was cancelled 2024-12-03T09:16:06,726 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:C 2024-12-03T09:16:06,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T09:16:06,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:06,778 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T09:16:06,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:06,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:06,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:06,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:06,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:06,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:06,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:06,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/0a16f93773634feb85dfc93ccf4e9008 is 50, key is test_row_0/A:col10/1733217366142/Put/seqid=0 2024-12-03T09:16:06,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742281_1457 (size=12301) 2024-12-03T09:16:06,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217426821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:06,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:06,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217426929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T09:16:07,121 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/668efbdb0db3492ba31e9cb290f3b25f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/668efbdb0db3492ba31e9cb290f3b25f 2024-12-03T09:16:07,124 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/A of 427f6f3f7c0d84ddc6f684fba93003f4 into 668efbdb0db3492ba31e9cb290f3b25f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:07,124 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:07,124 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/A, priority=13, startTime=1733217366703; duration=0sec 2024-12-03T09:16:07,125 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:07,125 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:A 2024-12-03T09:16:07,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217427139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:07,191 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/0a16f93773634feb85dfc93ccf4e9008 2024-12-03T09:16:07,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/9eb79f944b6449f4ab3698cf61d6d413 is 50, key is test_row_0/B:col10/1733217366142/Put/seqid=0 2024-12-03T09:16:07,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742282_1458 (size=12301) 2024-12-03T09:16:07,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217427450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:07,605 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/9eb79f944b6449f4ab3698cf61d6d413 2024-12-03T09:16:07,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/60ea7de9fe034453b4d246f458408f0e is 50, key is test_row_0/C:col10/1733217366142/Put/seqid=0 2024-12-03T09:16:07,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742283_1459 (size=12301) 2024-12-03T09:16:07,636 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/60ea7de9fe034453b4d246f458408f0e 2024-12-03T09:16:07,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/0a16f93773634feb85dfc93ccf4e9008 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/0a16f93773634feb85dfc93ccf4e9008 2024-12-03T09:16:07,656 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/0a16f93773634feb85dfc93ccf4e9008, entries=150, sequenceid=305, filesize=12.0 K 2024-12-03T09:16:07,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/9eb79f944b6449f4ab3698cf61d6d413 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/9eb79f944b6449f4ab3698cf61d6d413 2024-12-03T09:16:07,661 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/9eb79f944b6449f4ab3698cf61d6d413, entries=150, sequenceid=305, filesize=12.0 K 2024-12-03T09:16:07,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/60ea7de9fe034453b4d246f458408f0e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/60ea7de9fe034453b4d246f458408f0e 2024-12-03T09:16:07,671 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/60ea7de9fe034453b4d246f458408f0e, entries=150, sequenceid=305, filesize=12.0 K 2024-12-03T09:16:07,672 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 427f6f3f7c0d84ddc6f684fba93003f4 in 894ms, sequenceid=305, compaction requested=true 2024-12-03T09:16:07,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:07,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:07,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-03T09:16:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-03T09:16:07,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-03T09:16:07,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8820 sec 2024-12-03T09:16:07,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.8860 sec 2024-12-03T09:16:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:07,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T09:16:07,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:07,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:07,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:07,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:07,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:07,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:07,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/9cba89fff700401d8bf4a5f017463d3a is 50, key is test_row_1/A:col10/1733217366820/Put/seqid=0 2024-12-03T09:16:07,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742284_1460 (size=12297) 2024-12-03T09:16:07,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/9cba89fff700401d8bf4a5f017463d3a 2024-12-03T09:16:07,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/cd735d799c28475fa728ef7804c34252 is 50, key is test_row_1/B:col10/1733217366820/Put/seqid=0 2024-12-03T09:16:07,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742285_1461 (size=9857) 2024-12-03T09:16:07,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/cd735d799c28475fa728ef7804c34252 2024-12-03T09:16:07,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c3cbdc793de44e1a9002633e38fa96b9 is 50, key is test_row_1/C:col10/1733217366820/Put/seqid=0 2024-12-03T09:16:07,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742286_1462 (size=9857) 2024-12-03T09:16:07,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c3cbdc793de44e1a9002633e38fa96b9 2024-12-03T09:16:07,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/9cba89fff700401d8bf4a5f017463d3a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/9cba89fff700401d8bf4a5f017463d3a 2024-12-03T09:16:07,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/9cba89fff700401d8bf4a5f017463d3a, entries=150, sequenceid=317, filesize=12.0 K 2024-12-03T09:16:07,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/cd735d799c28475fa728ef7804c34252 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/cd735d799c28475fa728ef7804c34252 2024-12-03T09:16:07,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/cd735d799c28475fa728ef7804c34252, entries=100, sequenceid=317, filesize=9.6 K 2024-12-03T09:16:07,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217367756, max timestamp=1733217367777Num uniques 859; 2024-12-03T09:16:07,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217367756, max timestamp=1733217367777Num uniques 859; 2024-12-03T09:16:07,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:07,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:07,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c3cbdc793de44e1a9002633e38fa96b9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c3cbdc793de44e1a9002633e38fa96b9 2024-12-03T09:16:07,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217367756, max timestamp=1733217367777Num uniques 859; 2024-12-03T09:16:07,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:07,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c3cbdc793de44e1a9002633e38fa96b9, entries=100, sequenceid=317, filesize=9.6 K 2024-12-03T09:16:07,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=26.84 KB/27480 for 427f6f3f7c0d84ddc6f684fba93003f4 in 103ms, sequenceid=317, compaction requested=true 2024-12-03T09:16:07,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:07,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:07,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:07,785 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:07,785 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:07,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:07,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:07,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:07,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:07,786 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:07,786 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37445 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:07,786 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/B is initiating minor compaction (all files) 2024-12-03T09:16:07,786 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/A is initiating minor compaction (all files) 2024-12-03T09:16:07,786 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/A in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:07,786 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/B in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:07,786 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/7be56f38947d418ab29850d91611fea0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/9eb79f944b6449f4ab3698cf61d6d413, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/cd735d799c28475fa728ef7804c34252] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=34.2 K 2024-12-03T09:16:07,786 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/668efbdb0db3492ba31e9cb290f3b25f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/0a16f93773634feb85dfc93ccf4e9008, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/9cba89fff700401d8bf4a5f017463d3a] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=36.6 K 2024-12-03T09:16:07,787 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 668efbdb0db3492ba31e9cb290f3b25f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733217365836 2024-12-03T09:16:07,787 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7be56f38947d418ab29850d91611fea0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733217365836 2024-12-03T09:16:07,788 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a16f93773634feb85dfc93ccf4e9008, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733217366085 2024-12-03T09:16:07,788 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cba89fff700401d8bf4a5f017463d3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217366804 2024-12-03T09:16:07,789 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eb79f944b6449f4ab3698cf61d6d413, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733217366085 2024-12-03T09:16:07,789 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting cd735d799c28475fa728ef7804c34252, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217366820 2024-12-03T09:16:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:07,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-03T09:16:07,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:07,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:07,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:07,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:07,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:07,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:07,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/99f58200a90e49bc9ef5bbd2603482b0 is 50, key is test_row_0/A:col10/1733217367789/Put/seqid=0 2024-12-03T09:16:07,798 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#A#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:07,798 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/9e99a12139de47cb87af8ff2b4a8adff is 50, key is test_row_0/A:col10/1733217366142/Put/seqid=0 2024-12-03T09:16:07,803 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#B#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:07,803 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/8569648e3b7e4dbdab130d33a3299fb6 is 50, key is test_row_0/B:col10/1733217366142/Put/seqid=0 2024-12-03T09:16:07,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742287_1463 (size=19621) 2024-12-03T09:16:07,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/99f58200a90e49bc9ef5bbd2603482b0 2024-12-03T09:16:07,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742288_1464 (size=12949) 2024-12-03T09:16:07,832 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/9e99a12139de47cb87af8ff2b4a8adff as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/9e99a12139de47cb87af8ff2b4a8adff 2024-12-03T09:16:07,838 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/A of 427f6f3f7c0d84ddc6f684fba93003f4 into 9e99a12139de47cb87af8ff2b4a8adff(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:07,838 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:07,838 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/A, priority=13, startTime=1733217367785; duration=0sec 2024-12-03T09:16:07,838 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:07,838 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:A 2024-12-03T09:16:07,838 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:07,839 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47088 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:07,840 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/C is initiating minor compaction (all files) 2024-12-03T09:16:07,840 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/C in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:07,840 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c4b6a51e5f5c4d1ca61241a4f1cba64b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/67ee1264d9b04374a5ad4b7635034bed, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/60ea7de9fe034453b4d246f458408f0e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c3cbdc793de44e1a9002633e38fa96b9] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=46.0 K 2024-12-03T09:16:07,840 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4b6a51e5f5c4d1ca61241a4f1cba64b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733217364621 2024-12-03T09:16:07,841 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67ee1264d9b04374a5ad4b7635034bed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733217365836 2024-12-03T09:16:07,841 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60ea7de9fe034453b4d246f458408f0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733217366085 2024-12-03T09:16:07,841 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3cbdc793de44e1a9002633e38fa96b9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217366820 2024-12-03T09:16:07,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742289_1465 (size=12949) 2024-12-03T09:16:07,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/c498d545d4774719982abcfba6107842 is 50, key is test_row_0/B:col10/1733217367789/Put/seqid=0 2024-12-03T09:16:07,846 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/8569648e3b7e4dbdab130d33a3299fb6 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8569648e3b7e4dbdab130d33a3299fb6 2024-12-03T09:16:07,852 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#C#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:07,852 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/14216c71162f4c288b4bf6f33aa901f9 is 50, key is test_row_0/C:col10/1733217366142/Put/seqid=0 2024-12-03T09:16:07,854 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/B of 427f6f3f7c0d84ddc6f684fba93003f4 into 8569648e3b7e4dbdab130d33a3299fb6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:07,854 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:07,854 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/B, priority=13, startTime=1733217367785; duration=0sec 2024-12-03T09:16:07,854 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:07,854 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:B 2024-12-03T09:16:07,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742290_1466 (size=12301) 2024-12-03T09:16:07,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217367827, max timestamp=1733217367865Num uniques 859; 2024-12-03T09:16:07,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:16:07,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217367827, max timestamp=1733217367865Num uniques 859; 2024-12-03T09:16:07,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:16:07,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217367827, max timestamp=1733217367865Num uniques 859; 2024-12-03T09:16:07,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:16:07,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742291_1467 (size=12915) 2024-12-03T09:16:07,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:07,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217427904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:07,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733217367902, max timestamp=1733217367931Num uniques 1068; 2024-12-03T09:16:07,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T09:16:07,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733217367902, max timestamp=1733217367931Num uniques 1068; 2024-12-03T09:16:07,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733217367902, max timestamp=1733217367931Num uniques 1068; 2024-12-03T09:16:07,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T09:16:07,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T09:16:07,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217427968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:07,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:07,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217427969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:07,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217427970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:07,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:07,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:07,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217427971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:07,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:07,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217428039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217428080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217428089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217428089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217428089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217428249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/c498d545d4774719982abcfba6107842 2024-12-03T09:16:08,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c9e4e200e46f45059fad2eb43c3a18fe is 50, key is test_row_0/C:col10/1733217367789/Put/seqid=0 2024-12-03T09:16:08,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742292_1468 (size=12301) 2024-12-03T09:16:08,277 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/14216c71162f4c288b4bf6f33aa901f9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/14216c71162f4c288b4bf6f33aa901f9 2024-12-03T09:16:08,281 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/C of 427f6f3f7c0d84ddc6f684fba93003f4 into 14216c71162f4c288b4bf6f33aa901f9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:08,281 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:08,281 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/C, priority=12, startTime=1733217367785; duration=0sec 2024-12-03T09:16:08,281 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:08,281 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:C 2024-12-03T09:16:08,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217428290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217428299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217428299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217428299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217428559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217428603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217428609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217428609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217428609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:08,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:08,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c9e4e200e46f45059fad2eb43c3a18fe 2024-12-03T09:16:08,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/99f58200a90e49bc9ef5bbd2603482b0 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/99f58200a90e49bc9ef5bbd2603482b0 2024-12-03T09:16:08,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/99f58200a90e49bc9ef5bbd2603482b0, entries=300, sequenceid=342, filesize=19.2 K 2024-12-03T09:16:08,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/c498d545d4774719982abcfba6107842 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/c498d545d4774719982abcfba6107842 2024-12-03T09:16:08,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/c498d545d4774719982abcfba6107842, entries=150, sequenceid=342, filesize=12.0 K 2024-12-03T09:16:08,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c9e4e200e46f45059fad2eb43c3a18fe as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c9e4e200e46f45059fad2eb43c3a18fe 2024-12-03T09:16:08,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c9e4e200e46f45059fad2eb43c3a18fe, entries=150, sequenceid=342, filesize=12.0 K 2024-12-03T09:16:08,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 427f6f3f7c0d84ddc6f684fba93003f4 in 895ms, sequenceid=342, compaction requested=false 2024-12-03T09:16:08,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T09:16:08,949 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:08,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-03T09:16:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T09:16:08,952 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:08,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:08,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T09:16:09,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=140.89 KB heapSize=404.77 KB 2024-12-03T09:16:09,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:09,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T09:16:09,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:09,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T09:16:09,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:09,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T09:16:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:09,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/d8eccd8580cd411291f06f6b901f5453 is 50, key is test_row_0/A:col10/1733217367967/Put/seqid=0 2024-12-03T09:16:09,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742293_1469 (size=14741) 2024-12-03T09:16:09,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:09,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:09,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:09,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:09,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217429119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217429119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:09,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217429119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:09,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217429119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:09,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217429123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:09,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217429239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:09,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:09,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T09:16:09,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:09,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:09,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:09,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:09,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217429449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:09,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/d8eccd8580cd411291f06f6b901f5453 2024-12-03T09:16:09,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/e932ceb1d51040f2af28df484609ceeb is 50, key is test_row_0/B:col10/1733217367967/Put/seqid=0 2024-12-03T09:16:09,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742294_1470 (size=12301) 2024-12-03T09:16:09,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/e932ceb1d51040f2af28df484609ceeb 2024-12-03T09:16:09,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/870cfee444a340ce9d15a3a5b096c54b is 50, key is test_row_0/C:col10/1733217367967/Put/seqid=0 2024-12-03T09:16:09,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:09,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:09,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:09,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:09,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T09:16:09,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742295_1471 (size=12301) 2024-12-03T09:16:09,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:09,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:09,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:09,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:09,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217429770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:09,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:09,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:09,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:09,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:09,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/870cfee444a340ce9d15a3a5b096c54b 2024-12-03T09:16:10,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/d8eccd8580cd411291f06f6b901f5453 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/d8eccd8580cd411291f06f6b901f5453 2024-12-03T09:16:10,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/d8eccd8580cd411291f06f6b901f5453, entries=200, sequenceid=400, filesize=14.4 K 2024-12-03T09:16:10,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/e932ceb1d51040f2af28df484609ceeb as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e932ceb1d51040f2af28df484609ceeb 2024-12-03T09:16:10,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:10,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:10,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e932ceb1d51040f2af28df484609ceeb, entries=150, sequenceid=400, filesize=12.0 K 2024-12-03T09:16:10,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/870cfee444a340ce9d15a3a5b096c54b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/870cfee444a340ce9d15a3a5b096c54b 2024-12-03T09:16:10,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/870cfee444a340ce9d15a3a5b096c54b, entries=150, sequenceid=400, filesize=12.0 K 2024-12-03T09:16:10,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~404.72 KB/414432, currentSize=40.25 KB/41220 for 427f6f3f7c0d84ddc6f684fba93003f4 in 971ms, sequenceid=400, compaction requested=true 2024-12-03T09:16:10,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:10,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:10,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:10,046 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:10,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:10,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:10,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:10,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T09:16:10,046 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:10,052 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:10,052 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:10,052 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/A is initiating minor compaction (all files) 2024-12-03T09:16:10,052 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/B is initiating minor compaction (all files) 2024-12-03T09:16:10,052 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/B in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,052 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/A in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,052 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8569648e3b7e4dbdab130d33a3299fb6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/c498d545d4774719982abcfba6107842, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e932ceb1d51040f2af28df484609ceeb] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=36.7 K 2024-12-03T09:16:10,052 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/9e99a12139de47cb87af8ff2b4a8adff, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/99f58200a90e49bc9ef5bbd2603482b0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/d8eccd8580cd411291f06f6b901f5453] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=46.2 K 2024-12-03T09:16:10,053 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e99a12139de47cb87af8ff2b4a8adff, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217366142 2024-12-03T09:16:10,053 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99f58200a90e49bc9ef5bbd2603482b0, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733217367773 2024-12-03T09:16:10,053 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8569648e3b7e4dbdab130d33a3299fb6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217366142 2024-12-03T09:16:10,054 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8eccd8580cd411291f06f6b901f5453, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733217367967 2024-12-03T09:16:10,054 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting c498d545d4774719982abcfba6107842, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733217367774 2024-12-03T09:16:10,055 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting e932ceb1d51040f2af28df484609ceeb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733217367967 2024-12-03T09:16:10,066 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#B#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:10,067 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/029912df44594ba1b0ccf38b563c7cab is 50, key is test_row_0/B:col10/1733217367967/Put/seqid=0 2024-12-03T09:16:10,074 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#A#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:10,075 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/dad64435384e4587a89bd7667be3213b is 50, key is test_row_0/A:col10/1733217367967/Put/seqid=0 2024-12-03T09:16:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T09:16:10,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742297_1473 (size=13051) 2024-12-03T09:16:10,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742296_1472 (size=13051) 2024-12-03T09:16:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:10,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T09:16:10,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:10,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:10,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:10,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:10,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:10,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:10,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/1a8aaa913aad4ce597b736e5bf82ace4 is 50, key is test_row_0/A:col10/1733217369115/Put/seqid=0 2024-12-03T09:16:10,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742298_1474 (size=14741) 2024-12-03T09:16:10,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:10,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:10,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217370195, max timestamp=1733217370256Num uniques 859; 2024-12-03T09:16:10,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:10,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217370195, max timestamp=1733217370256Num uniques 859; 2024-12-03T09:16:10,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217370195, max timestamp=1733217370256Num uniques 859; 2024-12-03T09:16:10,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:10,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:10,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733217370295, max timestamp=1733217370316Num uniques 1018; 2024-12-03T09:16:10,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-03T09:16:10,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733217370295, max timestamp=1733217370316Num uniques 1018; 2024-12-03T09:16:10,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733217370295, max timestamp=1733217370316Num uniques 1018; 2024-12-03T09:16:10,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-03T09:16:10,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-03T09:16:10,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:10,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:10,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217430372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217430378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217430391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217430393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217430393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:10,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:10,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217430500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217430501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,532 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/dad64435384e4587a89bd7667be3213b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/dad64435384e4587a89bd7667be3213b 2024-12-03T09:16:10,538 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/A of 427f6f3f7c0d84ddc6f684fba93003f4 into dad64435384e4587a89bd7667be3213b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:10,538 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:10,538 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/A, priority=13, startTime=1733217370045; duration=0sec 2024-12-03T09:16:10,539 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:10,539 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:A 2024-12-03T09:16:10,539 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:10,541 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:10,541 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/C is initiating minor compaction (all files) 2024-12-03T09:16:10,541 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/C in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,541 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/14216c71162f4c288b4bf6f33aa901f9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c9e4e200e46f45059fad2eb43c3a18fe, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/870cfee444a340ce9d15a3a5b096c54b] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=36.6 K 2024-12-03T09:16:10,542 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14216c71162f4c288b4bf6f33aa901f9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217366142 2024-12-03T09:16:10,543 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9e4e200e46f45059fad2eb43c3a18fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733217367774 2024-12-03T09:16:10,544 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/029912df44594ba1b0ccf38b563c7cab as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/029912df44594ba1b0ccf38b563c7cab 2024-12-03T09:16:10,544 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 870cfee444a340ce9d15a3a5b096c54b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733217367967 2024-12-03T09:16:10,549 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/B of 427f6f3f7c0d84ddc6f684fba93003f4 into 029912df44594ba1b0ccf38b563c7cab(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:10,549 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:10,549 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/B, priority=13, startTime=1733217370046; duration=0sec 2024-12-03T09:16:10,549 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:10,549 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:B 2024-12-03T09:16:10,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217430530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217430530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217430530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,556 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#C#compaction#401 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:10,557 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/2912a082b26c4423a639d44b1aa285c6 is 50, key is test_row_0/C:col10/1733217367967/Put/seqid=0 2024-12-03T09:16:10,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/1a8aaa913aad4ce597b736e5bf82ace4 2024-12-03T09:16:10,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742299_1475 (size=13017) 2024-12-03T09:16:10,585 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/2912a082b26c4423a639d44b1aa285c6 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/2912a082b26c4423a639d44b1aa285c6 2024-12-03T09:16:10,590 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/C of 427f6f3f7c0d84ddc6f684fba93003f4 into 2912a082b26c4423a639d44b1aa285c6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:10,590 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:10,590 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/C, priority=13, startTime=1733217370046; duration=0sec 2024-12-03T09:16:10,590 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:10,590 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:C 2024-12-03T09:16:10,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/6d958ea6d2f846d18d1099b8e19b6391 is 50, key is test_row_0/B:col10/1733217369115/Put/seqid=0 2024-12-03T09:16:10,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:10,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:10,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742300_1476 (size=12301) 2024-12-03T09:16:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217430720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217430720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,734 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217430759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217430759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217430759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:10,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:10,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:10,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:10,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:10,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:10,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:11,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217431040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:11,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217431040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:11,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/6d958ea6d2f846d18d1099b8e19b6391 2024-12-03T09:16:11,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/5fd409bb7cd34c0a9dac4dced54974e3 is 50, key is test_row_0/C:col10/1733217369115/Put/seqid=0 2024-12-03T09:16:11,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217431090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:11,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:11,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217431090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217431092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T09:16:11,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:11,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:11,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742301_1477 (size=12301) 2024-12-03T09:16:11,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/5fd409bb7cd34c0a9dac4dced54974e3 2024-12-03T09:16:11,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/1a8aaa913aad4ce597b736e5bf82ace4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/1a8aaa913aad4ce597b736e5bf82ace4 2024-12-03T09:16:11,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/1a8aaa913aad4ce597b736e5bf82ace4, entries=200, sequenceid=410, filesize=14.4 K 2024-12-03T09:16:11,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/6d958ea6d2f846d18d1099b8e19b6391 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/6d958ea6d2f846d18d1099b8e19b6391 2024-12-03T09:16:11,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/6d958ea6d2f846d18d1099b8e19b6391, entries=150, sequenceid=410, filesize=12.0 K 2024-12-03T09:16:11,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/5fd409bb7cd34c0a9dac4dced54974e3 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/5fd409bb7cd34c0a9dac4dced54974e3 2024-12-03T09:16:11,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/5fd409bb7cd34c0a9dac4dced54974e3, entries=150, sequenceid=410, filesize=12.0 K 2024-12-03T09:16:11,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 427f6f3f7c0d84ddc6f684fba93003f4 in 1014ms, sequenceid=410, compaction requested=false 2024-12-03T09:16:11,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:11,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-03T09:16:11,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:11,266 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=134.18 KB heapSize=379.45 KB 2024-12-03T09:16:11,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:11,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:11,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:11,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:11,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:11,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:11,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/7d7a11b755244e038003666180d467af is 50, key is test_row_0/A:col10/1733217370389/Put/seqid=0 2024-12-03T09:16:11,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742302_1478 (size=12301) 2024-12-03T09:16:11,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:11,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217431600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:11,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:11,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217431602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:11,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217431603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:11,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:11,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:11,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217431610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:11,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:11,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217431610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:11,708 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/7d7a11b755244e038003666180d467af 2024-12-03T09:16:11,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217431709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:11,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/de878d74142b4f87b56aa941c82f290a is 50, key is test_row_0/B:col10/1733217370389/Put/seqid=0 2024-12-03T09:16:11,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742303_1479 (size=12301) 2024-12-03T09:16:11,726 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/de878d74142b4f87b56aa941c82f290a 2024-12-03T09:16:11,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217431719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:11,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/54fbaa3988df4baaada8e490aa470192 is 50, key is test_row_0/C:col10/1733217370389/Put/seqid=0 2024-12-03T09:16:11,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217431730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:11,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:11,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:11,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742304_1480 (size=12301) 2024-12-03T09:16:11,773 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/54fbaa3988df4baaada8e490aa470192 2024-12-03T09:16:11,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/7d7a11b755244e038003666180d467af as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/7d7a11b755244e038003666180d467af 2024-12-03T09:16:11,782 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/7d7a11b755244e038003666180d467af, entries=150, sequenceid=466, filesize=12.0 K 2024-12-03T09:16:11,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/de878d74142b4f87b56aa941c82f290a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/de878d74142b4f87b56aa941c82f290a 2024-12-03T09:16:11,787 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/de878d74142b4f87b56aa941c82f290a, entries=150, sequenceid=466, filesize=12.0 K 2024-12-03T09:16:11,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/54fbaa3988df4baaada8e490aa470192 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/54fbaa3988df4baaada8e490aa470192 2024-12-03T09:16:11,811 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/54fbaa3988df4baaada8e490aa470192, entries=150, sequenceid=466, filesize=12.0 K 2024-12-03T09:16:11,814 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~379.41 KB/388512, currentSize=46.96 KB/48090 for 427f6f3f7c0d84ddc6f684fba93003f4 in 548ms, sequenceid=466, compaction requested=true 2024-12-03T09:16:11,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:11,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:11,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-03T09:16:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-03T09:16:11,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-03T09:16:11,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8690 sec 2024-12-03T09:16:11,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.8750 sec 2024-12-03T09:16:11,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T09:16:11,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:11,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:11,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:11,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:11,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:11,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:11,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/6ee2d0b7af994241a6fd99b0b5750889 is 50, key is test_row_0/A:col10/1733217371565/Put/seqid=0 2024-12-03T09:16:11,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742305_1481 (size=14741) 2024-12-03T09:16:11,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/6ee2d0b7af994241a6fd99b0b5750889 2024-12-03T09:16:11,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/8ad71eb84cb349569067e3c5d879db60 is 50, key is test_row_0/B:col10/1733217371565/Put/seqid=0 2024-12-03T09:16:11,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742306_1482 (size=12301) 2024-12-03T09:16:11,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/8ad71eb84cb349569067e3c5d879db60 2024-12-03T09:16:11,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/1d213fe48081453ab659b0c7606fb73e is 50, key is test_row_0/C:col10/1733217371565/Put/seqid=0 2024-12-03T09:16:12,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217371972, max timestamp=1733217372016Num uniques 859; 2024-12-03T09:16:12,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217371972, max timestamp=1733217372016Num uniques 859; 2024-12-03T09:16:12,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:16:12,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:16:12,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733217371972, max timestamp=1733217372016Num uniques 859; 2024-12-03T09:16:12,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T09:16:12,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742307_1483 (size=12301) 2024-12-03T09:16:12,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/1d213fe48081453ab659b0c7606fb73e 2024-12-03T09:16:12,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/6ee2d0b7af994241a6fd99b0b5750889 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6ee2d0b7af994241a6fd99b0b5750889 2024-12-03T09:16:12,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6ee2d0b7af994241a6fd99b0b5750889, entries=200, sequenceid=477, filesize=14.4 K 2024-12-03T09:16:12,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/8ad71eb84cb349569067e3c5d879db60 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8ad71eb84cb349569067e3c5d879db60 2024-12-03T09:16:12,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8ad71eb84cb349569067e3c5d879db60, entries=150, sequenceid=477, filesize=12.0 K 2024-12-03T09:16:12,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/1d213fe48081453ab659b0c7606fb73e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/1d213fe48081453ab659b0c7606fb73e 2024-12-03T09:16:12,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/1d213fe48081453ab659b0c7606fb73e, entries=150, sequenceid=477, filesize=12.0 K 2024-12-03T09:16:12,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=80.51 KB/82440 for 427f6f3f7c0d84ddc6f684fba93003f4 in 143ms, sequenceid=477, compaction requested=true 2024-12-03T09:16:12,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:12,066 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:12,066 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:12,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:12,067 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:12,067 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/A is initiating minor compaction (all files) 2024-12-03T09:16:12,067 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/A in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:12,067 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/dad64435384e4587a89bd7667be3213b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/1a8aaa913aad4ce597b736e5bf82ace4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/7d7a11b755244e038003666180d467af, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6ee2d0b7af994241a6fd99b0b5750889] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=53.5 K 2024-12-03T09:16:12,067 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:12,068 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/B is initiating minor compaction (all files) 2024-12-03T09:16:12,068 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/B in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:12,068 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/029912df44594ba1b0ccf38b563c7cab, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/6d958ea6d2f846d18d1099b8e19b6391, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/de878d74142b4f87b56aa941c82f290a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8ad71eb84cb349569067e3c5d879db60] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=48.8 K 2024-12-03T09:16:12,068 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting dad64435384e4587a89bd7667be3213b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733217367967 2024-12-03T09:16:12,068 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 029912df44594ba1b0ccf38b563c7cab, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733217367967 2024-12-03T09:16:12,068 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a8aaa913aad4ce597b736e5bf82ace4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733217369108 2024-12-03T09:16:12,068 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d958ea6d2f846d18d1099b8e19b6391, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733217369115 2024-12-03T09:16:12,069 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d7a11b755244e038003666180d467af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1733217370353 2024-12-03T09:16:12,069 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting de878d74142b4f87b56aa941c82f290a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1733217370353 2024-12-03T09:16:12,069 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ee2d0b7af994241a6fd99b0b5750889, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733217371565 2024-12-03T09:16:12,069 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ad71eb84cb349569067e3c5d879db60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733217371565 2024-12-03T09:16:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:12,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=87.22 KB heapSize=240.84 KB 2024-12-03T09:16:12,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:12,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:12,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:12,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:12,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:12,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:12,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/cfead98ef05a4b84a74626e8d9697f17 is 50, key is test_row_0/A:col10/1733217372037/Put/seqid=0 2024-12-03T09:16:12,099 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:12,099 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/8bf3528e1fb449b2a3023c7176d32a79 is 50, key is test_row_0/B:col10/1733217371565/Put/seqid=0 2024-12-03T09:16:12,114 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#A#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:12,115 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/2945d548deff49ceaacb539f482601d1 is 50, key is test_row_0/A:col10/1733217371565/Put/seqid=0 2024-12-03T09:16:12,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742309_1485 (size=13187) 2024-12-03T09:16:12,161 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/8bf3528e1fb449b2a3023c7176d32a79 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8bf3528e1fb449b2a3023c7176d32a79 2024-12-03T09:16:12,170 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/B of 427f6f3f7c0d84ddc6f684fba93003f4 into 8bf3528e1fb449b2a3023c7176d32a79(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:12,170 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:12,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742308_1484 (size=14741) 2024-12-03T09:16:12,172 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/B, priority=12, startTime=1733217372066; duration=0sec 2024-12-03T09:16:12,172 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:12,172 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:B 2024-12-03T09:16:12,173 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:12,174 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:12,174 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/C is initiating minor compaction (all files) 2024-12-03T09:16:12,174 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/C in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:12,174 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/2912a082b26c4423a639d44b1aa285c6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/5fd409bb7cd34c0a9dac4dced54974e3, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/54fbaa3988df4baaada8e490aa470192, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/1d213fe48081453ab659b0c7606fb73e] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=48.8 K 2024-12-03T09:16:12,175 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2912a082b26c4423a639d44b1aa285c6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733217367967 2024-12-03T09:16:12,175 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fd409bb7cd34c0a9dac4dced54974e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1733217369115 2024-12-03T09:16:12,175 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 54fbaa3988df4baaada8e490aa470192, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1733217370353 2024-12-03T09:16:12,176 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d213fe48081453ab659b0c7606fb73e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733217371565 2024-12-03T09:16:12,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742310_1486 (size=13187) 2024-12-03T09:16:12,215 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#C#compaction#413 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:12,216 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/6080c0c59b42494bab219709bc55a47c is 50, key is test_row_0/C:col10/1733217371565/Put/seqid=0 2024-12-03T09:16:12,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217432181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,222 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/2945d548deff49ceaacb539f482601d1 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/2945d548deff49ceaacb539f482601d1 2024-12-03T09:16:12,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:12,229 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/A of 427f6f3f7c0d84ddc6f684fba93003f4 into 2945d548deff49ceaacb539f482601d1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:12,229 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:12,229 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/A, priority=12, startTime=1733217372066; duration=0sec 2024-12-03T09:16:12,229 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:12,229 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:A 2024-12-03T09:16:12,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742311_1487 (size=13153) 2024-12-03T09:16:12,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217432224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217432229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:12,261 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/6080c0c59b42494bab219709bc55a47c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/6080c0c59b42494bab219709bc55a47c 2024-12-03T09:16:12,265 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/C of 427f6f3f7c0d84ddc6f684fba93003f4 into 6080c0c59b42494bab219709bc55a47c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:12,265 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:12,265 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/C, priority=12, startTime=1733217372066; duration=0sec 2024-12-03T09:16:12,266 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:12,266 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:C 2024-12-03T09:16:12,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217432329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:12,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217432349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217432349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:12,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217432550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:12,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/cfead98ef05a4b84a74626e8d9697f17 2024-12-03T09:16:12,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217432571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:12,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217432571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:12,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/45e90fbda12141e0a945560394c99b7a is 50, key is test_row_0/B:col10/1733217372037/Put/seqid=0 2024-12-03T09:16:12,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742312_1488 (size=12301) 2024-12-03T09:16:12,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217432630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217432630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,636 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 427f6f3f7c0d84ddc6f684fba93003f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:12,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:12,638 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 427f6f3f7c0d84ddc6f684fba93003f4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:12,800 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T09:16:12,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217432869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:12,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217432879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:12,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217432889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:12,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:12,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:13,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/45e90fbda12141e0a945560394c99b7a 2024-12-03T09:16:13,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/5ed126bb671640b08814d456d0f65b58 is 50, key is test_row_0/C:col10/1733217372037/Put/seqid=0 2024-12-03T09:16:13,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742313_1489 (size=12301) 2024-12-03T09:16:13,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T09:16:13,130 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:13,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:13,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-03T09:16:13,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T09:16:13,134 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:13,135 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:13,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:13,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T09:16:13,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T09:16:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:13,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:13,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:13,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217433380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:13,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:13,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217433389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:13,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217433409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:13,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:13,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:13,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:13,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/5ed126bb671640b08814d456d0f65b58 2024-12-03T09:16:13,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T09:16:13,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:13,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:13,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:13,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:13,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:13,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/cfead98ef05a4b84a74626e8d9697f17 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/cfead98ef05a4b84a74626e8d9697f17 2024-12-03T09:16:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T09:16:13,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/cfead98ef05a4b84a74626e8d9697f17, entries=200, sequenceid=506, filesize=14.4 K 2024-12-03T09:16:13,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/45e90fbda12141e0a945560394c99b7a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/45e90fbda12141e0a945560394c99b7a 2024-12-03T09:16:13,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/45e90fbda12141e0a945560394c99b7a, entries=150, sequenceid=506, filesize=12.0 K 2024-12-03T09:16:13,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/5ed126bb671640b08814d456d0f65b58 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/5ed126bb671640b08814d456d0f65b58 2024-12-03T09:16:13,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/5ed126bb671640b08814d456d0f65b58, entries=150, sequenceid=506, filesize=12.0 K 2024-12-03T09:16:13,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~240.80 KB/246576, currentSize=100.63 KB/103050 for 427f6f3f7c0d84ddc6f684fba93003f4 in 1386ms, sequenceid=506, compaction requested=false 2024-12-03T09:16:13,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:13,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-03T09:16:13,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:13,597 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T09:16:13,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:13,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:13,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:13,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:13,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:13,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:13,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/afa8c8da98f142909992fe6b0b84890f is 50, key is test_row_0/A:col10/1733217372176/Put/seqid=0 2024-12-03T09:16:13,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742314_1490 (size=12301) 2024-12-03T09:16:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T09:16:14,007 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/afa8c8da98f142909992fe6b0b84890f 2024-12-03T09:16:14,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/36a537a28d2449deb841b12d8a5650e6 is 50, key is test_row_0/B:col10/1733217372176/Put/seqid=0 2024-12-03T09:16:14,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742315_1491 (size=12301) 2024-12-03T09:16:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T09:16:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:14,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:14,434 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/36a537a28d2449deb841b12d8a5650e6 2024-12-03T09:16:14,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/0125b3139ca14118a84a34d107755d14 is 50, key is test_row_0/C:col10/1733217372176/Put/seqid=0 2024-12-03T09:16:14,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742316_1492 (size=12301) 2024-12-03T09:16:14,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217434429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:14,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217434430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:14,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217434431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:14,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217434549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:14,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217434549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:14,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:14,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217434549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:14,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38096 deadline: 1733217434649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:14,654 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 427f6f3f7c0d84ddc6f684fba93003f4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:14,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38088 deadline: 1733217434659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:14,665 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 427f6f3f7c0d84ddc6f684fba93003f4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:14,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:14,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217434759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217434759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:14,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217434759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:14,842 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/0125b3139ca14118a84a34d107755d14 2024-12-03T09:16:14,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/afa8c8da98f142909992fe6b0b84890f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/afa8c8da98f142909992fe6b0b84890f 2024-12-03T09:16:14,848 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/afa8c8da98f142909992fe6b0b84890f, entries=150, sequenceid=527, filesize=12.0 K 2024-12-03T09:16:14,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/36a537a28d2449deb841b12d8a5650e6 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/36a537a28d2449deb841b12d8a5650e6 2024-12-03T09:16:14,852 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/36a537a28d2449deb841b12d8a5650e6, entries=150, sequenceid=527, filesize=12.0 K 2024-12-03T09:16:14,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/0125b3139ca14118a84a34d107755d14 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/0125b3139ca14118a84a34d107755d14 2024-12-03T09:16:14,855 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/0125b3139ca14118a84a34d107755d14, entries=150, sequenceid=527, filesize=12.0 K 2024-12-03T09:16:14,856 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 427f6f3f7c0d84ddc6f684fba93003f4 in 1259ms, sequenceid=527, compaction requested=true 2024-12-03T09:16:14,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:14,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:14,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-03T09:16:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-03T09:16:14,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-03T09:16:14,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7220 sec 2024-12-03T09:16:14,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.7280 sec 2024-12-03T09:16:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:15,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T09:16:15,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:15,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:15,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:15,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:15,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:15,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:15,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/14b2f87c7143442fa07de6c031c0a2ab is 50, key is test_row_0/A:col10/1733217374426/Put/seqid=0 2024-12-03T09:16:15,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742317_1493 (size=14741) 2024-12-03T09:16:15,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217435108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:15,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217435109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:15,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217435114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:15,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217435229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:15,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217435229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:15,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:15,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:15,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217435229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:15,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:15,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T09:16:15,279 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:15,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-03T09:16:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T09:16:15,289 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:15,290 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:15,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T09:16:15,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T09:16:15,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:15,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:15,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:15,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:15,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:15,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217435439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:15,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217435440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:15,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217435440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:15,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/14b2f87c7143442fa07de6c031c0a2ab 2024-12-03T09:16:15,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/bbc51a4135d84ffbbde0b5c15630e892 is 50, key is test_row_0/B:col10/1733217374426/Put/seqid=0 2024-12-03T09:16:15,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742318_1494 (size=12301) 2024-12-03T09:16:15,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/bbc51a4135d84ffbbde0b5c15630e892 2024-12-03T09:16:15,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/22b7b68039f14b6c839fe6d84562cdaf is 50, key is test_row_0/C:col10/1733217374426/Put/seqid=0 2024-12-03T09:16:15,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742319_1495 (size=12301) 2024-12-03T09:16:15,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T09:16:15,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:15,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:15,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:15,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:15,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T09:16:15,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T09:16:15,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:15,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:15,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:15,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:15,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:15,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38076 deadline: 1733217435749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:15,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38078 deadline: 1733217435749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:15,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:15,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38110 deadline: 1733217435749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 , the old value is region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., hostname=699c9baf3d29,42271,1733217224446, seqNum=2 because the exception is null or not the one we care about 2024-12-03T09:16:15,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T09:16:15,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:15,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:15,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:15,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:15,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:15,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/22b7b68039f14b6c839fe6d84562cdaf 2024-12-03T09:16:15,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/14b2f87c7143442fa07de6c031c0a2ab as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/14b2f87c7143442fa07de6c031c0a2ab 2024-12-03T09:16:15,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/14b2f87c7143442fa07de6c031c0a2ab, entries=200, sequenceid=542, filesize=14.4 K 2024-12-03T09:16:15,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/bbc51a4135d84ffbbde0b5c15630e892 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/bbc51a4135d84ffbbde0b5c15630e892 2024-12-03T09:16:15,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/bbc51a4135d84ffbbde0b5c15630e892, entries=150, sequenceid=542, filesize=12.0 K 2024-12-03T09:16:15,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/22b7b68039f14b6c839fe6d84562cdaf as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/22b7b68039f14b6c839fe6d84562cdaf 2024-12-03T09:16:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T09:16:15,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/22b7b68039f14b6c839fe6d84562cdaf, entries=150, sequenceid=542, filesize=12.0 K 2024-12-03T09:16:15,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 427f6f3f7c0d84ddc6f684fba93003f4 in 850ms, sequenceid=542, compaction requested=true 2024-12-03T09:16:15,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:15,925 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:15,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:15,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:15,926 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:15,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:15,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:15,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 427f6f3f7c0d84ddc6f684fba93003f4:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:15,926 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:15,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:15,927 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/A is initiating minor compaction (all files) 2024-12-03T09:16:15,927 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/A in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:15,927 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/2945d548deff49ceaacb539f482601d1, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/cfead98ef05a4b84a74626e8d9697f17, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/afa8c8da98f142909992fe6b0b84890f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/14b2f87c7143442fa07de6c031c0a2ab] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=53.7 K 2024-12-03T09:16:15,927 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2945d548deff49ceaacb539f482601d1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733217371565 2024-12-03T09:16:15,928 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:15,928 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/B is initiating minor compaction (all files) 2024-12-03T09:16:15,928 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/B in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:15,928 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8bf3528e1fb449b2a3023c7176d32a79, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/45e90fbda12141e0a945560394c99b7a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/36a537a28d2449deb841b12d8a5650e6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/bbc51a4135d84ffbbde0b5c15630e892] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=48.9 K 2024-12-03T09:16:15,928 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfead98ef05a4b84a74626e8d9697f17, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1733217372034 2024-12-03T09:16:15,928 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bf3528e1fb449b2a3023c7176d32a79, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733217371565 2024-12-03T09:16:15,929 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting afa8c8da98f142909992fe6b0b84890f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1733217372169 2024-12-03T09:16:15,929 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 45e90fbda12141e0a945560394c99b7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1733217372034 2024-12-03T09:16:15,929 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 36a537a28d2449deb841b12d8a5650e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1733217372169 2024-12-03T09:16:15,929 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14b2f87c7143442fa07de6c031c0a2ab, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1733217374426 2024-12-03T09:16:15,929 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting bbc51a4135d84ffbbde0b5c15630e892, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1733217374426 2024-12-03T09:16:15,942 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#B#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:15,942 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#A#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:15,943 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/0d048d854f134023aec06cf94defc4f2 is 50, key is test_row_0/B:col10/1733217374426/Put/seqid=0 2024-12-03T09:16:15,943 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/902aba5473c045218b3ddaa143b85763 is 50, key is test_row_0/A:col10/1733217374426/Put/seqid=0 2024-12-03T09:16:15,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742320_1496 (size=13323) 2024-12-03T09:16:15,992 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/902aba5473c045218b3ddaa143b85763 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/902aba5473c045218b3ddaa143b85763 2024-12-03T09:16:16,000 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/A of 427f6f3f7c0d84ddc6f684fba93003f4 into 902aba5473c045218b3ddaa143b85763(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:16,000 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:16,000 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/A, priority=12, startTime=1733217375925; duration=0sec 2024-12-03T09:16:16,000 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:16,000 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:A 2024-12-03T09:16:16,000 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:16,001 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:16,001 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 427f6f3f7c0d84ddc6f684fba93003f4/C is initiating minor compaction (all files) 2024-12-03T09:16:16,002 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 427f6f3f7c0d84ddc6f684fba93003f4/C in TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:16,002 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/6080c0c59b42494bab219709bc55a47c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/5ed126bb671640b08814d456d0f65b58, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/0125b3139ca14118a84a34d107755d14, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/22b7b68039f14b6c839fe6d84562cdaf] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp, totalSize=48.9 K 2024-12-03T09:16:16,003 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6080c0c59b42494bab219709bc55a47c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733217371565 2024-12-03T09:16:16,003 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ed126bb671640b08814d456d0f65b58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=506, earliestPutTs=1733217372034 2024-12-03T09:16:16,003 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0125b3139ca14118a84a34d107755d14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1733217372169 2024-12-03T09:16:16,003 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22b7b68039f14b6c839fe6d84562cdaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1733217374426 2024-12-03T09:16:16,016 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 427f6f3f7c0d84ddc6f684fba93003f4#C#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:16,016 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/0339161f59974ba0a03877249791c445 is 50, key is test_row_0/C:col10/1733217374426/Put/seqid=0 2024-12-03T09:16:16,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742321_1497 (size=13323) 2024-12-03T09:16:16,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742322_1498 (size=13289) 2024-12-03T09:16:16,043 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/0339161f59974ba0a03877249791c445 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/0339161f59974ba0a03877249791c445 2024-12-03T09:16:16,048 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/C of 427f6f3f7c0d84ddc6f684fba93003f4 into 0339161f59974ba0a03877249791c445(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:16,048 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:16,048 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/C, priority=12, startTime=1733217375926; duration=0sec 2024-12-03T09:16:16,048 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:16,048 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:C 2024-12-03T09:16:16,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-03T09:16:16,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:16,056 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T09:16:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:16,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:16,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/facc1fe1158f4aed88c2b7ec565d77e9 is 50, key is test_row_0/A:col10/1733217375100/Put/seqid=0 2024-12-03T09:16:16,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742323_1499 (size=12301) 2024-12-03T09:16:16,124 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/facc1fe1158f4aed88c2b7ec565d77e9 2024-12-03T09:16:16,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/2cc0ee62d87c498e92cd4f203ef4706c is 50, key is test_row_0/B:col10/1733217375100/Put/seqid=0 2024-12-03T09:16:16,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742324_1500 (size=12301) 2024-12-03T09:16:16,168 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/2cc0ee62d87c498e92cd4f203ef4706c 2024-12-03T09:16:16,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c01b88b5cc3e440b827e27d244decaf3 is 50, key is test_row_0/C:col10/1733217375100/Put/seqid=0 2024-12-03T09:16:16,201 INFO [Thread-1867 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1867. 2024-12-03T09:16:16,201 DEBUG [Thread-1867 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:16,201 DEBUG [Thread-1867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,201 DEBUG [Thread-1867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:16,204 INFO [Thread-1865 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1865. 2024-12-03T09:16:16,205 DEBUG [Thread-1865 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:16,205 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,205 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:16,206 INFO [Thread-1863 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1863. 2024-12-03T09:16:16,206 DEBUG [Thread-1863 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:16,206 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,206 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:16,212 INFO [Thread-1859 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1859. 2024-12-03T09:16:16,212 DEBUG [Thread-1859 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:16,212 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,212 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:16,217 INFO [Thread-1861 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1861. 2024-12-03T09:16:16,218 DEBUG [Thread-1861 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:16,218 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,218 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:16,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742325_1501 (size=12301) 2024-12-03T09:16:16,249 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c01b88b5cc3e440b827e27d244decaf3 2024-12-03T09:16:16,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/facc1fe1158f4aed88c2b7ec565d77e9 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/facc1fe1158f4aed88c2b7ec565d77e9 2024-12-03T09:16:16,259 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/facc1fe1158f4aed88c2b7ec565d77e9, entries=150, sequenceid=561, filesize=12.0 K 2024-12-03T09:16:16,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/2cc0ee62d87c498e92cd4f203ef4706c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/2cc0ee62d87c498e92cd4f203ef4706c 2024-12-03T09:16:16,263 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/2cc0ee62d87c498e92cd4f203ef4706c, entries=150, sequenceid=561, filesize=12.0 K 2024-12-03T09:16:16,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/c01b88b5cc3e440b827e27d244decaf3 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c01b88b5cc3e440b827e27d244decaf3 2024-12-03T09:16:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:16,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. as already flushing 2024-12-03T09:16:16,272 INFO [Thread-1852 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1852. 2024-12-03T09:16:16,272 INFO [Thread-1850 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1850. 2024-12-03T09:16:16,273 DEBUG [Thread-1852 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:16,273 DEBUG [Thread-1850 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:16,273 DEBUG [Thread-1852 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,273 DEBUG [Thread-1850 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,273 DEBUG [Thread-1850 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,273 DEBUG [Thread-1852 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:16,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:16,278 INFO [Thread-1856 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1856. 2024-12-03T09:16:16,278 DEBUG [Thread-1856 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:16,278 DEBUG [Thread-1856 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,278 DEBUG [Thread-1856 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:16,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:16,279 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c01b88b5cc3e440b827e27d244decaf3, entries=150, sequenceid=561, filesize=12.0 K 2024-12-03T09:16:16,280 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for 427f6f3f7c0d84ddc6f684fba93003f4 in 224ms, sequenceid=561, compaction requested=false 2024-12-03T09:16:16,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:16,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:16,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-03T09:16:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-03T09:16:16,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-03T09:16:16,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 992 msec 2024-12-03T09:16:16,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.0100 sec 2024-12-03T09:16:16,426 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/0d048d854f134023aec06cf94defc4f2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/0d048d854f134023aec06cf94defc4f2 2024-12-03T09:16:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T09:16:16,430 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:16,432 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 427f6f3f7c0d84ddc6f684fba93003f4/B of 427f6f3f7c0d84ddc6f684fba93003f4 into 0d048d854f134023aec06cf94defc4f2(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:16,432 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 427f6f3f7c0d84ddc6f684fba93003f4: 2024-12-03T09:16:16,432 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., storeName=427f6f3f7c0d84ddc6f684fba93003f4/B, priority=12, startTime=1733217375926; duration=0sec 2024-12-03T09:16:16,432 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:16,432 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 427f6f3f7c0d84ddc6f684fba93003f4:B 2024-12-03T09:16:18,683 INFO [Thread-1854 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1854. 2024-12-03T09:16:18,683 DEBUG [Thread-1854 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:18,683 DEBUG [Thread-1854 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:18,683 DEBUG [Thread-1854 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:18,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:18,693 INFO [Thread-1848 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1848. 2024-12-03T09:16:18,693 DEBUG [Thread-1848 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:18,693 DEBUG [Thread-1848 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:18,693 DEBUG [Thread-1848 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:18,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1357 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4071 rows 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1361 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4083 rows 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1355 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4062 rows 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1346 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4034 rows 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1348 2024-12-03T09:16:18,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4044 rows 2024-12-03T09:16:18,694 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T09:16:18,694 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:16:18,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:18,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:18,695 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:18,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T09:16:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:18,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T09:16:18,701 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217378700"}]},"ts":"1733217378700"} 2024-12-03T09:16:18,703 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T09:16:18,703 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T09:16:18,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T09:16:18,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=427f6f3f7c0d84ddc6f684fba93003f4, UNASSIGN}] 2024-12-03T09:16:18,707 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=427f6f3f7c0d84ddc6f684fba93003f4, UNASSIGN 2024-12-03T09:16:18,708 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=427f6f3f7c0d84ddc6f684fba93003f4, regionState=CLOSING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:18,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=427f6f3f7c0d84ddc6f684fba93003f4, UNASSIGN because future has completed 2024-12-03T09:16:18,710 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T09:16:18,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:16:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T09:16:18,864 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:18,864 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T09:16:18,865 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 427f6f3f7c0d84ddc6f684fba93003f4, disabling compactions & flushes 2024-12-03T09:16:18,865 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:18,865 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:18,865 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. after waiting 0 ms 2024-12-03T09:16:18,865 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:18,865 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing 427f6f3f7c0d84ddc6f684fba93003f4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T09:16:18,866 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=A 2024-12-03T09:16:18,866 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:18,866 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=B 2024-12-03T09:16:18,866 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:18,866 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 427f6f3f7c0d84ddc6f684fba93003f4, store=C 2024-12-03T09:16:18,866 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:18,873 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/4b60987133d74d72bb0133901a003034 is 50, key is test_row_0/A:col10/1733217376271/Put/seqid=0 2024-12-03T09:16:18,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742326_1502 (size=12301) 2024-12-03T09:16:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T09:16:19,278 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/4b60987133d74d72bb0133901a003034 2024-12-03T09:16:19,284 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/0ed24e9eab6b4917b85f751853cfa47f is 50, key is test_row_0/B:col10/1733217376271/Put/seqid=0 2024-12-03T09:16:19,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742327_1503 (size=12301) 2024-12-03T09:16:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T09:16:19,689 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/0ed24e9eab6b4917b85f751853cfa47f 2024-12-03T09:16:19,695 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/504ac95dedc44662a29cb320f82f96a5 is 50, key is test_row_0/C:col10/1733217376271/Put/seqid=0 2024-12-03T09:16:19,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742328_1504 (size=12301) 2024-12-03T09:16:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T09:16:20,098 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/504ac95dedc44662a29cb320f82f96a5 2024-12-03T09:16:20,103 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/A/4b60987133d74d72bb0133901a003034 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/4b60987133d74d72bb0133901a003034 2024-12-03T09:16:20,106 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/4b60987133d74d72bb0133901a003034, entries=150, sequenceid=570, filesize=12.0 K 2024-12-03T09:16:20,107 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/B/0ed24e9eab6b4917b85f751853cfa47f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/0ed24e9eab6b4917b85f751853cfa47f 2024-12-03T09:16:20,110 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/0ed24e9eab6b4917b85f751853cfa47f, entries=150, sequenceid=570, filesize=12.0 K 2024-12-03T09:16:20,111 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/.tmp/C/504ac95dedc44662a29cb320f82f96a5 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/504ac95dedc44662a29cb320f82f96a5 2024-12-03T09:16:20,115 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/504ac95dedc44662a29cb320f82f96a5, entries=150, sequenceid=570, filesize=12.0 K 2024-12-03T09:16:20,115 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 427f6f3f7c0d84ddc6f684fba93003f4 in 1250ms, sequenceid=570, compaction requested=true 2024-12-03T09:16:20,116 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/1ca8cb46cfc247e9af8b5e231d021acb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/3681826c492a4016876b0b467aef27b6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/ce7357a32bd842078b701df5c66fc582, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/5d6afe50d2e2457f80aeebe378a96506, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/b55ec9bcff394568b9f36ea2612daba3, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/50bcae6267f74be384adbe22a5714856, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/a16df14416e04e2f91d0905b802436fe, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/a4fc4a0349dd4d029c9b004fc027e870, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/7d5752d999034937843e6c8aafffb1b9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6b0c3f87c83d4370acef4adbf6ffa240, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6b3e9dc2373c49a7945b91d35d884897, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/b28f0a22eff04779bf380ce6cfcf9904, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/cfc1e512f3e04162815879bf5f135cb9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/d029ad9d90704553ae7dec5cfbf1054b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/55f1ebf18ea04868ab267cb09fd5afaf, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/809076f9f9f849749826d1b5ee8cf135, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/668efbdb0db3492ba31e9cb290f3b25f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/0a16f93773634feb85dfc93ccf4e9008, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/9e99a12139de47cb87af8ff2b4a8adff, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/9cba89fff700401d8bf4a5f017463d3a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/99f58200a90e49bc9ef5bbd2603482b0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/d8eccd8580cd411291f06f6b901f5453, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/dad64435384e4587a89bd7667be3213b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/1a8aaa913aad4ce597b736e5bf82ace4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/7d7a11b755244e038003666180d467af, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6ee2d0b7af994241a6fd99b0b5750889, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/2945d548deff49ceaacb539f482601d1, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/cfead98ef05a4b84a74626e8d9697f17, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/afa8c8da98f142909992fe6b0b84890f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/14b2f87c7143442fa07de6c031c0a2ab] to archive 2024-12-03T09:16:20,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:16:20,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/1ca8cb46cfc247e9af8b5e231d021acb to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/1ca8cb46cfc247e9af8b5e231d021acb 2024-12-03T09:16:20,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/3681826c492a4016876b0b467aef27b6 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/3681826c492a4016876b0b467aef27b6 2024-12-03T09:16:20,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/ce7357a32bd842078b701df5c66fc582 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/ce7357a32bd842078b701df5c66fc582 2024-12-03T09:16:20,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/5d6afe50d2e2457f80aeebe378a96506 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/5d6afe50d2e2457f80aeebe378a96506 2024-12-03T09:16:20,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/b55ec9bcff394568b9f36ea2612daba3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/b55ec9bcff394568b9f36ea2612daba3 2024-12-03T09:16:20,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/50bcae6267f74be384adbe22a5714856 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/50bcae6267f74be384adbe22a5714856 2024-12-03T09:16:20,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/a16df14416e04e2f91d0905b802436fe to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/a16df14416e04e2f91d0905b802436fe 2024-12-03T09:16:20,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/a4fc4a0349dd4d029c9b004fc027e870 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/a4fc4a0349dd4d029c9b004fc027e870 2024-12-03T09:16:20,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/7d5752d999034937843e6c8aafffb1b9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/7d5752d999034937843e6c8aafffb1b9 2024-12-03T09:16:20,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6b0c3f87c83d4370acef4adbf6ffa240 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6b0c3f87c83d4370acef4adbf6ffa240 2024-12-03T09:16:20,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6b3e9dc2373c49a7945b91d35d884897 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6b3e9dc2373c49a7945b91d35d884897 2024-12-03T09:16:20,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/b28f0a22eff04779bf380ce6cfcf9904 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/b28f0a22eff04779bf380ce6cfcf9904 2024-12-03T09:16:20,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/cfc1e512f3e04162815879bf5f135cb9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/cfc1e512f3e04162815879bf5f135cb9 2024-12-03T09:16:20,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/d029ad9d90704553ae7dec5cfbf1054b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/d029ad9d90704553ae7dec5cfbf1054b 2024-12-03T09:16:20,132 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/55f1ebf18ea04868ab267cb09fd5afaf to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/55f1ebf18ea04868ab267cb09fd5afaf 2024-12-03T09:16:20,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/809076f9f9f849749826d1b5ee8cf135 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/809076f9f9f849749826d1b5ee8cf135 2024-12-03T09:16:20,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/668efbdb0db3492ba31e9cb290f3b25f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/668efbdb0db3492ba31e9cb290f3b25f 2024-12-03T09:16:20,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/0a16f93773634feb85dfc93ccf4e9008 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/0a16f93773634feb85dfc93ccf4e9008 2024-12-03T09:16:20,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/9e99a12139de47cb87af8ff2b4a8adff to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/9e99a12139de47cb87af8ff2b4a8adff 2024-12-03T09:16:20,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/9cba89fff700401d8bf4a5f017463d3a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/9cba89fff700401d8bf4a5f017463d3a 2024-12-03T09:16:20,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/99f58200a90e49bc9ef5bbd2603482b0 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/99f58200a90e49bc9ef5bbd2603482b0 2024-12-03T09:16:20,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/d8eccd8580cd411291f06f6b901f5453 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/d8eccd8580cd411291f06f6b901f5453 2024-12-03T09:16:20,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/dad64435384e4587a89bd7667be3213b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/dad64435384e4587a89bd7667be3213b 2024-12-03T09:16:20,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/1a8aaa913aad4ce597b736e5bf82ace4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/1a8aaa913aad4ce597b736e5bf82ace4 2024-12-03T09:16:20,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/7d7a11b755244e038003666180d467af to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/7d7a11b755244e038003666180d467af 2024-12-03T09:16:20,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6ee2d0b7af994241a6fd99b0b5750889 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/6ee2d0b7af994241a6fd99b0b5750889 2024-12-03T09:16:20,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/2945d548deff49ceaacb539f482601d1 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/2945d548deff49ceaacb539f482601d1 2024-12-03T09:16:20,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/cfead98ef05a4b84a74626e8d9697f17 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/cfead98ef05a4b84a74626e8d9697f17 2024-12-03T09:16:20,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/afa8c8da98f142909992fe6b0b84890f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/afa8c8da98f142909992fe6b0b84890f 2024-12-03T09:16:20,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/14b2f87c7143442fa07de6c031c0a2ab to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/14b2f87c7143442fa07de6c031c0a2ab 2024-12-03T09:16:20,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/76f029183f8a4d4b8f824442040a760a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e9892308d2784422a3bfc10d805d8f92, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/b736c05fd58f4b4db82e8585893ae180, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/3c09d7ed2e124e84a4e272d9951f8288, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/cde91bfc7c2348e184eaaf0b31106f7f, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/9eb529a09acf4806828a53a6816758f4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/1e2ca974b2e548d89a6e2a171618a7b8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/243661c488da4ff5bd1a039682fbe4b7, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/3b0d5b9616084571a84e445d42d49574, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/09efcb2cbe3148c6aae06938aa8bf495, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/92f1a711295c4e0386e12376570ad12a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/1a55c23d25f544e8a9c64cf57e4fca41, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/68d6fb9d7dbf4d349f6476be012f5496, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/7da0bf9c46394df8bf1af01addb829eb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/b566e1161da242bf96b8fb55d3b55d19, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/7be56f38947d418ab29850d91611fea0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e41bf757d714421a9dfd239c46213d65, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/9eb79f944b6449f4ab3698cf61d6d413, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8569648e3b7e4dbdab130d33a3299fb6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/cd735d799c28475fa728ef7804c34252, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/c498d545d4774719982abcfba6107842, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/029912df44594ba1b0ccf38b563c7cab, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e932ceb1d51040f2af28df484609ceeb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/6d958ea6d2f846d18d1099b8e19b6391, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/de878d74142b4f87b56aa941c82f290a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8bf3528e1fb449b2a3023c7176d32a79, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8ad71eb84cb349569067e3c5d879db60, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/45e90fbda12141e0a945560394c99b7a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/36a537a28d2449deb841b12d8a5650e6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/bbc51a4135d84ffbbde0b5c15630e892] to archive 2024-12-03T09:16:20,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:16:20,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/76f029183f8a4d4b8f824442040a760a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/76f029183f8a4d4b8f824442040a760a 2024-12-03T09:16:20,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e9892308d2784422a3bfc10d805d8f92 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e9892308d2784422a3bfc10d805d8f92 2024-12-03T09:16:20,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/b736c05fd58f4b4db82e8585893ae180 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/b736c05fd58f4b4db82e8585893ae180 2024-12-03T09:16:20,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/3c09d7ed2e124e84a4e272d9951f8288 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/3c09d7ed2e124e84a4e272d9951f8288 2024-12-03T09:16:20,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/cde91bfc7c2348e184eaaf0b31106f7f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/cde91bfc7c2348e184eaaf0b31106f7f 2024-12-03T09:16:20,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/9eb529a09acf4806828a53a6816758f4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/9eb529a09acf4806828a53a6816758f4 2024-12-03T09:16:20,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/1e2ca974b2e548d89a6e2a171618a7b8 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/1e2ca974b2e548d89a6e2a171618a7b8 2024-12-03T09:16:20,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/243661c488da4ff5bd1a039682fbe4b7 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/243661c488da4ff5bd1a039682fbe4b7 2024-12-03T09:16:20,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/3b0d5b9616084571a84e445d42d49574 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/3b0d5b9616084571a84e445d42d49574 2024-12-03T09:16:20,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/09efcb2cbe3148c6aae06938aa8bf495 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/09efcb2cbe3148c6aae06938aa8bf495 2024-12-03T09:16:20,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/92f1a711295c4e0386e12376570ad12a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/92f1a711295c4e0386e12376570ad12a 2024-12-03T09:16:20,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/1a55c23d25f544e8a9c64cf57e4fca41 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/1a55c23d25f544e8a9c64cf57e4fca41 2024-12-03T09:16:20,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/68d6fb9d7dbf4d349f6476be012f5496 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/68d6fb9d7dbf4d349f6476be012f5496 2024-12-03T09:16:20,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/7da0bf9c46394df8bf1af01addb829eb to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/7da0bf9c46394df8bf1af01addb829eb 2024-12-03T09:16:20,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/b566e1161da242bf96b8fb55d3b55d19 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/b566e1161da242bf96b8fb55d3b55d19 2024-12-03T09:16:20,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/7be56f38947d418ab29850d91611fea0 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/7be56f38947d418ab29850d91611fea0 2024-12-03T09:16:20,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e41bf757d714421a9dfd239c46213d65 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e41bf757d714421a9dfd239c46213d65 2024-12-03T09:16:20,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/9eb79f944b6449f4ab3698cf61d6d413 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/9eb79f944b6449f4ab3698cf61d6d413 2024-12-03T09:16:20,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8569648e3b7e4dbdab130d33a3299fb6 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8569648e3b7e4dbdab130d33a3299fb6 2024-12-03T09:16:20,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/cd735d799c28475fa728ef7804c34252 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/cd735d799c28475fa728ef7804c34252 2024-12-03T09:16:20,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/c498d545d4774719982abcfba6107842 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/c498d545d4774719982abcfba6107842 2024-12-03T09:16:20,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/029912df44594ba1b0ccf38b563c7cab to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/029912df44594ba1b0ccf38b563c7cab 2024-12-03T09:16:20,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e932ceb1d51040f2af28df484609ceeb to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/e932ceb1d51040f2af28df484609ceeb 2024-12-03T09:16:20,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/6d958ea6d2f846d18d1099b8e19b6391 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/6d958ea6d2f846d18d1099b8e19b6391 2024-12-03T09:16:20,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/de878d74142b4f87b56aa941c82f290a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/de878d74142b4f87b56aa941c82f290a 2024-12-03T09:16:20,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8bf3528e1fb449b2a3023c7176d32a79 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8bf3528e1fb449b2a3023c7176d32a79 2024-12-03T09:16:20,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8ad71eb84cb349569067e3c5d879db60 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/8ad71eb84cb349569067e3c5d879db60 2024-12-03T09:16:20,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/45e90fbda12141e0a945560394c99b7a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/45e90fbda12141e0a945560394c99b7a 2024-12-03T09:16:20,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/36a537a28d2449deb841b12d8a5650e6 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/36a537a28d2449deb841b12d8a5650e6 2024-12-03T09:16:20,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/bbc51a4135d84ffbbde0b5c15630e892 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/bbc51a4135d84ffbbde0b5c15630e892 2024-12-03T09:16:20,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/4b1348939af449529c4eb76c682cf7f3, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/416004a2be904b83b38b4acb8cb84b92, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/1a05b00dd39b4b1f87ff104e401526b4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c085a8b9ba6b4b178d6a6579789bf416, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/68871811346341319e4e32fe25426031, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/b40ee0c06ad34f629692712a3363443a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/39ae1beba8cc4456ae841b1be31c4c23, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/b25d525f64a4439d9f122d5ad5944f0d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/f82163086e8941aeb3a797c0a4475ec4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/824a3394c4a2460b897f9a3ca00a05ab, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/268fd085de7940c8831342623bd0e684, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/76018d3b5ded4a64988897355fb2e234, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/81eb98f276bf4fd094122d2e6474f1eb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c4b6a51e5f5c4d1ca61241a4f1cba64b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/055c1ed5f27a4dce8a6f429b48a73e59, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/67ee1264d9b04374a5ad4b7635034bed, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/60ea7de9fe034453b4d246f458408f0e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/14216c71162f4c288b4bf6f33aa901f9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c3cbdc793de44e1a9002633e38fa96b9, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c9e4e200e46f45059fad2eb43c3a18fe, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/2912a082b26c4423a639d44b1aa285c6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/870cfee444a340ce9d15a3a5b096c54b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/5fd409bb7cd34c0a9dac4dced54974e3, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/54fbaa3988df4baaada8e490aa470192, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/6080c0c59b42494bab219709bc55a47c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/1d213fe48081453ab659b0c7606fb73e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/5ed126bb671640b08814d456d0f65b58, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/0125b3139ca14118a84a34d107755d14, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/22b7b68039f14b6c839fe6d84562cdaf] to archive 2024-12-03T09:16:20,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:16:20,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/4b1348939af449529c4eb76c682cf7f3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/4b1348939af449529c4eb76c682cf7f3 2024-12-03T09:16:20,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/416004a2be904b83b38b4acb8cb84b92 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/416004a2be904b83b38b4acb8cb84b92 2024-12-03T09:16:20,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/1a05b00dd39b4b1f87ff104e401526b4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/1a05b00dd39b4b1f87ff104e401526b4 2024-12-03T09:16:20,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c085a8b9ba6b4b178d6a6579789bf416 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c085a8b9ba6b4b178d6a6579789bf416 2024-12-03T09:16:20,180 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/68871811346341319e4e32fe25426031 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/68871811346341319e4e32fe25426031 2024-12-03T09:16:20,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/b40ee0c06ad34f629692712a3363443a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/b40ee0c06ad34f629692712a3363443a 2024-12-03T09:16:20,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/39ae1beba8cc4456ae841b1be31c4c23 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/39ae1beba8cc4456ae841b1be31c4c23 2024-12-03T09:16:20,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/b25d525f64a4439d9f122d5ad5944f0d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/b25d525f64a4439d9f122d5ad5944f0d 2024-12-03T09:16:20,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/f82163086e8941aeb3a797c0a4475ec4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/f82163086e8941aeb3a797c0a4475ec4 2024-12-03T09:16:20,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/824a3394c4a2460b897f9a3ca00a05ab to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/824a3394c4a2460b897f9a3ca00a05ab 2024-12-03T09:16:20,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/268fd085de7940c8831342623bd0e684 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/268fd085de7940c8831342623bd0e684 2024-12-03T09:16:20,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/76018d3b5ded4a64988897355fb2e234 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/76018d3b5ded4a64988897355fb2e234 2024-12-03T09:16:20,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/81eb98f276bf4fd094122d2e6474f1eb to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/81eb98f276bf4fd094122d2e6474f1eb 2024-12-03T09:16:20,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c4b6a51e5f5c4d1ca61241a4f1cba64b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c4b6a51e5f5c4d1ca61241a4f1cba64b 2024-12-03T09:16:20,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/055c1ed5f27a4dce8a6f429b48a73e59 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/055c1ed5f27a4dce8a6f429b48a73e59 2024-12-03T09:16:20,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/67ee1264d9b04374a5ad4b7635034bed to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/67ee1264d9b04374a5ad4b7635034bed 2024-12-03T09:16:20,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/60ea7de9fe034453b4d246f458408f0e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/60ea7de9fe034453b4d246f458408f0e 2024-12-03T09:16:20,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/14216c71162f4c288b4bf6f33aa901f9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/14216c71162f4c288b4bf6f33aa901f9 2024-12-03T09:16:20,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c3cbdc793de44e1a9002633e38fa96b9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c3cbdc793de44e1a9002633e38fa96b9 2024-12-03T09:16:20,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c9e4e200e46f45059fad2eb43c3a18fe to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c9e4e200e46f45059fad2eb43c3a18fe 2024-12-03T09:16:20,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/2912a082b26c4423a639d44b1aa285c6 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/2912a082b26c4423a639d44b1aa285c6 2024-12-03T09:16:20,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/870cfee444a340ce9d15a3a5b096c54b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/870cfee444a340ce9d15a3a5b096c54b 2024-12-03T09:16:20,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/5fd409bb7cd34c0a9dac4dced54974e3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/5fd409bb7cd34c0a9dac4dced54974e3 2024-12-03T09:16:20,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/54fbaa3988df4baaada8e490aa470192 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/54fbaa3988df4baaada8e490aa470192 2024-12-03T09:16:20,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/6080c0c59b42494bab219709bc55a47c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/6080c0c59b42494bab219709bc55a47c 2024-12-03T09:16:20,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/1d213fe48081453ab659b0c7606fb73e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/1d213fe48081453ab659b0c7606fb73e 2024-12-03T09:16:20,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/5ed126bb671640b08814d456d0f65b58 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/5ed126bb671640b08814d456d0f65b58 2024-12-03T09:16:20,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/0125b3139ca14118a84a34d107755d14 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/0125b3139ca14118a84a34d107755d14 2024-12-03T09:16:20,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/22b7b68039f14b6c839fe6d84562cdaf to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/22b7b68039f14b6c839fe6d84562cdaf 2024-12-03T09:16:20,203 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/recovered.edits/573.seqid, newMaxSeqId=573, maxSeqId=1 2024-12-03T09:16:20,203 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. 2024-12-03T09:16:20,204 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 427f6f3f7c0d84ddc6f684fba93003f4: Waiting for close lock at 1733217378865Running coprocessor pre-close hooks at 1733217378865Disabling compacts and flushes for region at 1733217378865Disabling writes for close at 1733217378865Obtaining lock to block concurrent updates at 1733217378865Preparing flush snapshotting stores in 427f6f3f7c0d84ddc6f684fba93003f4 at 1733217378865Finished memstore snapshotting TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733217378866 (+1 ms)Flushing stores of TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4. at 1733217378868 (+2 ms)Flushing 427f6f3f7c0d84ddc6f684fba93003f4/A: creating writer at 1733217378868Flushing 427f6f3f7c0d84ddc6f684fba93003f4/A: appending metadata at 1733217378872 (+4 ms)Flushing 427f6f3f7c0d84ddc6f684fba93003f4/A: closing flushed file at 1733217378873 (+1 ms)Flushing 427f6f3f7c0d84ddc6f684fba93003f4/B: creating writer at 1733217379281 (+408 ms)Flushing 427f6f3f7c0d84ddc6f684fba93003f4/B: appending metadata at 1733217379283 (+2 ms)Flushing 427f6f3f7c0d84ddc6f684fba93003f4/B: closing flushed file at 1733217379283Flushing 427f6f3f7c0d84ddc6f684fba93003f4/C: creating writer at 1733217379692 (+409 ms)Flushing 427f6f3f7c0d84ddc6f684fba93003f4/C: appending metadata at 1733217379694 (+2 ms)Flushing 427f6f3f7c0d84ddc6f684fba93003f4/C: closing flushed file at 1733217379694Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b308b1d: reopening flushed file at 1733217380102 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9075f03: reopening flushed file at 1733217380106 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@586765aa: reopening flushed file at 1733217380111 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 427f6f3f7c0d84ddc6f684fba93003f4 in 1250ms, sequenceid=570, compaction requested=true at 1733217380115 (+4 ms)Writing region close event to WAL at 1733217380201 (+86 ms)Running coprocessor post-close hooks at 1733217380203 (+2 ms)Closed at 1733217380203 2024-12-03T09:16:20,205 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:20,206 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=427f6f3f7c0d84ddc6f684fba93003f4, regionState=CLOSED 2024-12-03T09:16:20,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:16:20,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-03T09:16:20,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 427f6f3f7c0d84ddc6f684fba93003f4, server=699c9baf3d29,42271,1733217224446 in 1.4970 sec 2024-12-03T09:16:20,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-12-03T09:16:20,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=427f6f3f7c0d84ddc6f684fba93003f4, UNASSIGN in 1.5040 sec 2024-12-03T09:16:20,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-03T09:16:20,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5070 sec 2024-12-03T09:16:20,213 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217380213"}]},"ts":"1733217380213"} 2024-12-03T09:16:20,214 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T09:16:20,214 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T09:16:20,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5180 sec 2024-12-03T09:16:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T09:16:20,861 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:20,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T09:16:20,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:20,866 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-03T09:16:20,867 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:20,872 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:20,875 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/recovered.edits] 2024-12-03T09:16:20,878 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/4b60987133d74d72bb0133901a003034 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/4b60987133d74d72bb0133901a003034 2024-12-03T09:16:20,879 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/902aba5473c045218b3ddaa143b85763 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/902aba5473c045218b3ddaa143b85763 2024-12-03T09:16:20,880 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/facc1fe1158f4aed88c2b7ec565d77e9 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/A/facc1fe1158f4aed88c2b7ec565d77e9 2024-12-03T09:16:20,882 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/0d048d854f134023aec06cf94defc4f2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/0d048d854f134023aec06cf94defc4f2 2024-12-03T09:16:20,883 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/0ed24e9eab6b4917b85f751853cfa47f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/0ed24e9eab6b4917b85f751853cfa47f 2024-12-03T09:16:20,884 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/2cc0ee62d87c498e92cd4f203ef4706c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/B/2cc0ee62d87c498e92cd4f203ef4706c 2024-12-03T09:16:20,887 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/0339161f59974ba0a03877249791c445 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/0339161f59974ba0a03877249791c445 2024-12-03T09:16:20,889 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/504ac95dedc44662a29cb320f82f96a5 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/504ac95dedc44662a29cb320f82f96a5 2024-12-03T09:16:20,890 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c01b88b5cc3e440b827e27d244decaf3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/C/c01b88b5cc3e440b827e27d244decaf3 2024-12-03T09:16:20,893 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/recovered.edits/573.seqid to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4/recovered.edits/573.seqid 2024-12-03T09:16:20,894 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/427f6f3f7c0d84ddc6f684fba93003f4 2024-12-03T09:16:20,894 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T09:16:20,897 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:20,900 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T09:16:20,903 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T09:16:20,904 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:20,904 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T09:16:20,905 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733217380904"}]},"ts":"9223372036854775807"} 2024-12-03T09:16:20,906 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T09:16:20,906 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 427f6f3f7c0d84ddc6f684fba93003f4, NAME => 'TestAcidGuarantees,,1733217355005.427f6f3f7c0d84ddc6f684fba93003f4.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T09:16:20,906 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T09:16:20,907 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733217380906"}]},"ts":"9223372036854775807"} 2024-12-03T09:16:20,908 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T09:16:20,909 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:20,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-12-03T09:16:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-03T09:16:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T09:16:20,970 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:20,985 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 248) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=560 (was 387) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12777 (was 13325) 2024-12-03T09:16:20,993 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=560, ProcessCount=11, AvailableMemoryMB=12775 2024-12-03T09:16:20,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T09:16:20,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T09:16:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:20,997 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T09:16:20,997 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:20,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-12-03T09:16:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-03T09:16:20,998 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T09:16:21,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742329_1505 (size=1061) 2024-12-03T09:16:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-03T09:16:21,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-03T09:16:21,406 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09 2024-12-03T09:16:21,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742330_1506 (size=53) 2024-12-03T09:16:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-03T09:16:21,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:16:21,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 750235cb928bf724493a7da0f57c7eb3, disabling compactions & flushes 2024-12-03T09:16:21,817 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:21,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:21,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. after waiting 0 ms 2024-12-03T09:16:21,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:21,817 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:21,817 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 750235cb928bf724493a7da0f57c7eb3: Waiting for close lock at 1733217381817Disabling compacts and flushes for region at 1733217381817Disabling writes for close at 1733217381817Writing region close event to WAL at 1733217381817Closed at 1733217381817 2024-12-03T09:16:21,818 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T09:16:21,818 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733217381818"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733217381818"}]},"ts":"1733217381818"} 2024-12-03T09:16:21,820 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T09:16:21,821 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T09:16:21,821 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217381821"}]},"ts":"1733217381821"} 2024-12-03T09:16:21,822 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T09:16:21,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, ASSIGN}] 2024-12-03T09:16:21,823 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, ASSIGN 2024-12-03T09:16:21,823 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, ASSIGN; state=OFFLINE, location=699c9baf3d29,42271,1733217224446; forceNewPlan=false, retain=false 2024-12-03T09:16:21,974 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=750235cb928bf724493a7da0f57c7eb3, regionState=OPENING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:21,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, ASSIGN because future has completed 2024-12-03T09:16:21,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:16:22,134 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:22,134 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:16:22,135 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:22,135 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:16:22,135 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:22,135 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:22,137 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:22,139 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:16:22,140 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 750235cb928bf724493a7da0f57c7eb3 columnFamilyName A 2024-12-03T09:16:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-03T09:16:22,140 DEBUG [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:22,141 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.HStore(327): Store=750235cb928bf724493a7da0f57c7eb3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:16:22,141 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:22,142 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:16:22,143 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 750235cb928bf724493a7da0f57c7eb3 columnFamilyName B 2024-12-03T09:16:22,143 DEBUG [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:22,143 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.HStore(327): Store=750235cb928bf724493a7da0f57c7eb3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:16:22,143 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:22,144 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:16:22,145 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 750235cb928bf724493a7da0f57c7eb3 columnFamilyName C 2024-12-03T09:16:22,145 DEBUG [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:22,145 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.HStore(327): Store=750235cb928bf724493a7da0f57c7eb3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:16:22,145 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:22,145 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:22,146 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:22,146 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:22,148 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:22,148 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:22,148 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T09:16:22,150 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:22,151 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:16:22,152 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened 750235cb928bf724493a7da0f57c7eb3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62641143, jitterRate=-0.06657423079013824}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T09:16:22,152 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:22,153 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for 750235cb928bf724493a7da0f57c7eb3: Running coprocessor pre-open hook at 1733217382135Writing region info on filesystem at 1733217382135Initializing all the Stores at 1733217382136 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217382136Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217382137 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217382137Cleaning up temporary data from old regions at 1733217382148 (+11 ms)Running coprocessor post-open hooks at 1733217382152 (+4 ms)Region opened successfully at 1733217382152 2024-12-03T09:16:22,153 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., pid=142, masterSystemTime=1733217382130 2024-12-03T09:16:22,155 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:22,155 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:22,156 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=750235cb928bf724493a7da0f57c7eb3, regionState=OPEN, openSeqNum=2, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:22,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:16:22,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-03T09:16:22,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 in 181 msec 2024-12-03T09:16:22,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-03T09:16:22,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, ASSIGN in 338 msec 2024-12-03T09:16:22,164 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T09:16:22,164 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217382164"}]},"ts":"1733217382164"} 2024-12-03T09:16:22,166 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T09:16:22,167 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T09:16:22,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-12-03T09:16:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-12-03T09:16:23,150 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:23,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44dc71c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:23,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:16:23,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:16:23,152 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:16:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:16:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:16:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@645d0111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:16:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:16:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:23,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:16:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bc379b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:23,154 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:16:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:16:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:16:23,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:16:23,157 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T09:16:23,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 699c9baf3d29,35781,1733217223715 2024-12-03T09:16:23,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@74a2faf4 2024-12-03T09:16:23,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T09:16:23,159 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T09:16:23,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T09:16:23,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T09:16:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:23,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742331_1507 (size=1097) 2024-12-03T09:16:23,572 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T09:16:23,572 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T09:16:23,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T09:16:23,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, REOPEN/MOVE}] 2024-12-03T09:16:23,577 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, REOPEN/MOVE 2024-12-03T09:16:23,577 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=750235cb928bf724493a7da0f57c7eb3, regionState=CLOSING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:23,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, REOPEN/MOVE because future has completed 2024-12-03T09:16:23,578 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T09:16:23,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:16:23,731 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:23,731 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T09:16:23,731 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 750235cb928bf724493a7da0f57c7eb3, disabling compactions & flushes 2024-12-03T09:16:23,731 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:23,731 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:23,731 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. after waiting 0 ms 2024-12-03T09:16:23,731 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:23,735 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T09:16:23,735 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:23,735 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 750235cb928bf724493a7da0f57c7eb3: Waiting for close lock at 1733217383731Running coprocessor pre-close hooks at 1733217383731Disabling compacts and flushes for region at 1733217383731Disabling writes for close at 1733217383731Writing region close event to WAL at 1733217383731Running coprocessor post-close hooks at 1733217383735 (+4 ms)Closed at 1733217383735 2024-12-03T09:16:23,735 WARN [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: 750235cb928bf724493a7da0f57c7eb3 to self. 2024-12-03T09:16:23,737 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:23,738 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=750235cb928bf724493a7da0f57c7eb3, regionState=CLOSED 2024-12-03T09:16:23,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:16:23,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-03T09:16:23,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 in 163 msec 2024-12-03T09:16:23,744 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, REOPEN/MOVE; state=CLOSED, location=699c9baf3d29,42271,1733217224446; forceNewPlan=false, retain=true 2024-12-03T09:16:23,895 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=750235cb928bf724493a7da0f57c7eb3, regionState=OPENING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:23,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, REOPEN/MOVE because future has completed 2024-12-03T09:16:23,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:16:24,053 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:24,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T09:16:24,053 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:16:24,053 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,054 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:16:24,054 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,054 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,056 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,057 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:16:24,057 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 750235cb928bf724493a7da0f57c7eb3 columnFamilyName A 2024-12-03T09:16:24,059 DEBUG [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:24,060 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.HStore(327): Store=750235cb928bf724493a7da0f57c7eb3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:16:24,060 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,061 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:16:24,061 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 750235cb928bf724493a7da0f57c7eb3 columnFamilyName B 2024-12-03T09:16:24,061 DEBUG [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:24,061 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.HStore(327): Store=750235cb928bf724493a7da0f57c7eb3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:16:24,061 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,062 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T09:16:24,062 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 750235cb928bf724493a7da0f57c7eb3 columnFamilyName C 2024-12-03T09:16:24,062 DEBUG [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:24,063 INFO [StoreOpener-750235cb928bf724493a7da0f57c7eb3-1 {}] regionserver.HStore(327): Store=750235cb928bf724493a7da0f57c7eb3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:16:24,063 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:24,063 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,063 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,064 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,065 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,065 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,066 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T09:16:24,067 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,067 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened 750235cb928bf724493a7da0f57c7eb3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67628042, jitterRate=0.0077363550662994385}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T09:16:24,067 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,068 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for 750235cb928bf724493a7da0f57c7eb3: Running coprocessor pre-open hook at 1733217384054Writing region info on filesystem at 1733217384054Initializing all the Stores at 1733217384055 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217384055Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217384056 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217384056Cleaning up temporary data from old regions at 1733217384065 (+9 ms)Running coprocessor post-open hooks at 1733217384067 (+2 ms)Region opened successfully at 1733217384068 (+1 ms) 2024-12-03T09:16:24,069 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., pid=147, masterSystemTime=1733217384048 2024-12-03T09:16:24,070 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:24,071 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:24,071 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=750235cb928bf724493a7da0f57c7eb3, regionState=OPEN, openSeqNum=5, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:16:24,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-12-03T09:16:24,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 in 178 msec 2024-12-03T09:16:24,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-03T09:16:24,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, REOPEN/MOVE in 500 msec 2024-12-03T09:16:24,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-03T09:16:24,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-12-03T09:16:24,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 920 msec 2024-12-03T09:16:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-03T09:16:24,083 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:24,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4770660d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,083 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:16:24,084 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:16:24,084 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:16:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:16:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:16:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c128bfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:16:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:16:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:24,085 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:16:24,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fafede6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a50324d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,086 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:16:24,086 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:16:24,087 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:16:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:16:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:16:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b8f999b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:16:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:16:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:24,088 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:16:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e82e6ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19303c6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:16:24,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:16:24,089 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:16:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:16:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:16:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f27261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:16:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:16:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:24,090 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:16:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e56eaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef3a8ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:16:24,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:16:24,092 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:16:24,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:16:24,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:16:24,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@181b206f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:16:24,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:16:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:24,093 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:16:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2613ed98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf0f1f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:16:24,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:16:24,094 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:16:24,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:16:24,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:16:24,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d0a585, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:16:24,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:16:24,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:24,096 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:16:24,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b0148c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1624acf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:16:24,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:16:24,097 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:16:24,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:16:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:16:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c46b9bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:16:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:16:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:24,099 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:16:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@693abd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47198352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:16:24,100 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:16:24,101 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:16:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:16:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:16:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50af15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:16:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:16:24,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:24,102 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:16:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae47485, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@288c2f93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,103 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:16:24,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:16:24,104 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:16:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:16:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:16:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b5ecff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:16:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:16:24,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:24,106 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:16:24,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a550ee0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7597ad2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:16:24,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:16:24,107 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:16:24,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:16:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:16:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@720b1f56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:16:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:16:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:24,109 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:16:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6346c032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dedb2d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,111 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,35781,-1 for getting cluster id 2024-12-03T09:16:24,111 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:16:24,112 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c252829b-0c30-4211-bed0-58579dd285df' 2024-12-03T09:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c252829b-0c30-4211-bed0-58579dd285df" 2024-12-03T09:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f574a05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,35781,-1] 2024-12-03T09:16:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:16:24,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:24,114 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:16:24,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b7aff81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:16:24,116 DEBUG [Thread-2282 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:16:24,116 DEBUG [Thread-2299 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:16:24,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:16:24,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:16:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:16:24,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:16:24,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:16:24,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:16:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:16:24,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:16:24,123 DEBUG [Thread-2288 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:16:24,124 DEBUG [Thread-2284 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:16:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:16:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:16:24,124 DEBUG [Thread-2286 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:16:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:16:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:16:24,125 DEBUG [Thread-2297 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:16:24,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:16:24,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:16:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:16:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:16:24,127 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:16:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:16:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:16:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:16:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:16:24,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:16:24,129 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:16:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:16:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T09:16:24,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:24,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:24,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:24,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:24,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:24,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:24,137 DEBUG [Thread-2295 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:16:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:16:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:16:24,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:16:24,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:16:24,143 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:24,143 DEBUG [Thread-2280 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:16:24,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:16:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-03T09:16:24,144 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:16:24,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T09:16:24,145 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:24,145 DEBUG [Thread-2291 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:16:24,146 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:24,146 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:24,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:16:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:16:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:16:24,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:16:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:16:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:16:24,155 DEBUG [Thread-2293 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:16:24,156 DEBUG [Thread-2293 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,42271,1733217224446, seqNum=-1] 2024-12-03T09:16:24,156 DEBUG [Thread-2293 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:16:24,157 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:16:24,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217384143, max timestamp=1733217384156Num uniques 859; 2024-12-03T09:16:24,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5] 2024-12-03T09:16:24,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217384143, max timestamp=1733217384157Num uniques 859; 2024-12-03T09:16:24,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733217384143, max timestamp=1733217384157Num uniques 859; 2024-12-03T09:16:24,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:24,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/699c9baf3d29:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T09:16:24,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039d405f5c378544bdb5087790a4428300_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_1/A:col10/1733217384131/Put/seqid=0 2024-12-03T09:16:24,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742332_1508 (size=9714) 2024-12-03T09:16:24,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217444174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217444176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217444177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217444179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217444184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T09:16:24,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217444289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217444289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217444290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217444290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217444290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T09:16:24,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:24,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:24,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:24,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:24,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:24,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T09:16:24,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:24,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:24,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:24,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:24,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T09:16:24,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217444500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217444500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217444500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217444500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217444500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,572 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:24,575 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039d405f5c378544bdb5087790a4428300_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039d405f5c378544bdb5087790a4428300_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,575 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/e82b08caef46427cb6158098b48a83d2, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:24,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/e82b08caef46427cb6158098b48a83d2 is 175, key is test_row_1/A:col10/1733217384131/Put/seqid=0 2024-12-03T09:16:24,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742333_1509 (size=22361) 2024-12-03T09:16:24,579 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/e82b08caef46427cb6158098b48a83d2 2024-12-03T09:16:24,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/1ad4d01f224843fb980ea0288d8960a4 is 50, key is test_row_1/B:col10/1733217384131/Put/seqid=0 2024-12-03T09:16:24,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742334_1510 (size=9657) 2024-12-03T09:16:24,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/1ad4d01f224843fb980ea0288d8960a4 2024-12-03T09:16:24,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T09:16:24,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:24,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:24,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:24,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:24,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:24,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/f3fd8e73e3624aaba310e6f2b93ba958 is 50, key is test_row_1/C:col10/1733217384131/Put/seqid=0 2024-12-03T09:16:24,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742335_1511 (size=9657) 2024-12-03T09:16:24,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/f3fd8e73e3624aaba310e6f2b93ba958 2024-12-03T09:16:24,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/e82b08caef46427cb6158098b48a83d2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e82b08caef46427cb6158098b48a83d2 2024-12-03T09:16:24,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e82b08caef46427cb6158098b48a83d2, entries=100, sequenceid=15, filesize=21.8 K 2024-12-03T09:16:24,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/1ad4d01f224843fb980ea0288d8960a4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/1ad4d01f224843fb980ea0288d8960a4 2024-12-03T09:16:24,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/1ad4d01f224843fb980ea0288d8960a4, entries=100, sequenceid=15, filesize=9.4 K 2024-12-03T09:16:24,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/f3fd8e73e3624aaba310e6f2b93ba958 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/f3fd8e73e3624aaba310e6f2b93ba958 2024-12-03T09:16:24,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/f3fd8e73e3624aaba310e6f2b93ba958, entries=100, sequenceid=15, filesize=9.4 K 2024-12-03T09:16:24,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 750235cb928bf724493a7da0f57c7eb3 in 515ms, sequenceid=15, compaction requested=false 2024-12-03T09:16:24,647 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T09:16:24,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:24,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T09:16:24,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:24,757 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T09:16:24,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:24,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:24,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:24,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:24,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:24,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T09:16:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T09:16:24,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036025f3e696f24f00bf992f5650efa16a_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217384177/Put/seqid=0 2024-12-03T09:16:24,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742336_1512 (size=12154) 2024-12-03T09:16:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:24,819 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036025f3e696f24f00bf992f5650efa16a_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036025f3e696f24f00bf992f5650efa16a_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:24,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/19d2aa9ea525465ea56c97a094568b7a, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:24,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/19d2aa9ea525465ea56c97a094568b7a is 175, key is test_row_0/A:col10/1733217384177/Put/seqid=0 2024-12-03T09:16:24,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217444820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217444824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217444824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217444825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217444826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742337_1513 (size=30955) 2024-12-03T09:16:24,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217444929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217444939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217444939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217444939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217444939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:24,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217445139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217445149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217445149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217445149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217445150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,247 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/19d2aa9ea525465ea56c97a094568b7a 2024-12-03T09:16:25,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/521efd4a88f04e01a6821463a9991b98 is 50, key is test_row_0/B:col10/1733217384177/Put/seqid=0 2024-12-03T09:16:25,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742338_1514 (size=12001) 2024-12-03T09:16:25,261 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/521efd4a88f04e01a6821463a9991b98 2024-12-03T09:16:25,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/ff08a28d9388422bb7b8178084d39faf is 50, key is test_row_0/C:col10/1733217384177/Put/seqid=0 2024-12-03T09:16:25,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T09:16:25,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742339_1515 (size=12001) 2024-12-03T09:16:25,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217445449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217445459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217445459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217445459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217445460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,685 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/ff08a28d9388422bb7b8178084d39faf 2024-12-03T09:16:25,716 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T09:16:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/19d2aa9ea525465ea56c97a094568b7a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/19d2aa9ea525465ea56c97a094568b7a 2024-12-03T09:16:25,787 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/19d2aa9ea525465ea56c97a094568b7a, entries=150, sequenceid=51, filesize=30.2 K 2024-12-03T09:16:25,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/521efd4a88f04e01a6821463a9991b98 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/521efd4a88f04e01a6821463a9991b98 2024-12-03T09:16:25,793 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/521efd4a88f04e01a6821463a9991b98, entries=150, sequenceid=51, filesize=11.7 K 2024-12-03T09:16:25,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/ff08a28d9388422bb7b8178084d39faf as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/ff08a28d9388422bb7b8178084d39faf 2024-12-03T09:16:25,798 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/ff08a28d9388422bb7b8178084d39faf, entries=150, sequenceid=51, filesize=11.7 K 2024-12-03T09:16:25,799 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 750235cb928bf724493a7da0f57c7eb3 in 1042ms, sequenceid=51, compaction requested=false 2024-12-03T09:16:25,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:25,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:25,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-03T09:16:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-03T09:16:25,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-03T09:16:25,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6550 sec 2024-12-03T09:16:25,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.6600 sec 2024-12-03T09:16:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:25,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T09:16:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:25,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036ac1aed9390c49a0882c6416d12910c0_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217384824/Put/seqid=0 2024-12-03T09:16:25,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742340_1516 (size=14594) 2024-12-03T09:16:25,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217445982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217445983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217445983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217445984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217445984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:25,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217446089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217446089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217446100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217446100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217446100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T09:16:26,290 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:26,290 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-03T09:16:26,291 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T09:16:26,292 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:26,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:26,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217446300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217446300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217446309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217446309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217446313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,385 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:26,387 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036ac1aed9390c49a0882c6416d12910c0_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036ac1aed9390c49a0882c6416d12910c0_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:26,389 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/9cda3555ebd64c4a9bff089d7fe701f8, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:26,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/9cda3555ebd64c4a9bff089d7fe701f8 is 175, key is test_row_0/A:col10/1733217384824/Put/seqid=0 2024-12-03T09:16:26,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742341_1517 (size=39549) 2024-12-03T09:16:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T09:16:26,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T09:16:26,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:26,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:26,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:26,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:26,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:26,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T09:16:26,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:26,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:26,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:26,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:26,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T09:16:26,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217446609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217446609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217446619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217446619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:26,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217446619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:26,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:26,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T09:16:26,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:26,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:26,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:26,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:26,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:26,793 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/9cda3555ebd64c4a9bff089d7fe701f8 2024-12-03T09:16:26,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/63e50e96bdc3478d8c730b87c05c8618 is 50, key is test_row_0/B:col10/1733217384824/Put/seqid=0 2024-12-03T09:16:26,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742342_1518 (size=12001) 2024-12-03T09:16:26,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/63e50e96bdc3478d8c730b87c05c8618 2024-12-03T09:16:26,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/8f2b1b0f688c43eca63311f45d025341 is 50, key is test_row_0/C:col10/1733217384824/Put/seqid=0 2024-12-03T09:16:26,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742343_1519 (size=12001) 2024-12-03T09:16:26,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/8f2b1b0f688c43eca63311f45d025341 2024-12-03T09:16:26,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/9cda3555ebd64c4a9bff089d7fe701f8 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/9cda3555ebd64c4a9bff089d7fe701f8 2024-12-03T09:16:26,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/9cda3555ebd64c4a9bff089d7fe701f8, entries=200, sequenceid=64, filesize=38.6 K 2024-12-03T09:16:26,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/63e50e96bdc3478d8c730b87c05c8618 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/63e50e96bdc3478d8c730b87c05c8618 2024-12-03T09:16:26,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/63e50e96bdc3478d8c730b87c05c8618, entries=150, sequenceid=64, filesize=11.7 K 2024-12-03T09:16:26,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/8f2b1b0f688c43eca63311f45d025341 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/8f2b1b0f688c43eca63311f45d025341 2024-12-03T09:16:26,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/8f2b1b0f688c43eca63311f45d025341, entries=150, sequenceid=64, filesize=11.7 K 2024-12-03T09:16:26,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 750235cb928bf724493a7da0f57c7eb3 in 880ms, sequenceid=64, compaction requested=true 2024-12-03T09:16:26,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:26,841 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:26,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:26,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:26,842 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:26,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:26,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:26,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:26,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:26,843 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:26,843 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/B is initiating minor compaction (all files) 2024-12-03T09:16:26,843 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/B in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:26,843 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/1ad4d01f224843fb980ea0288d8960a4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/521efd4a88f04e01a6821463a9991b98, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/63e50e96bdc3478d8c730b87c05c8618] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=32.9 K 2024-12-03T09:16:26,843 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:26,843 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/A is initiating minor compaction (all files) 2024-12-03T09:16:26,844 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/A in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:26,844 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e82b08caef46427cb6158098b48a83d2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/19d2aa9ea525465ea56c97a094568b7a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/9cda3555ebd64c4a9bff089d7fe701f8] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=90.7 K 2024-12-03T09:16:26,844 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:26,844 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e82b08caef46427cb6158098b48a83d2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/19d2aa9ea525465ea56c97a094568b7a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/9cda3555ebd64c4a9bff089d7fe701f8] 2024-12-03T09:16:26,845 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ad4d01f224843fb980ea0288d8960a4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733217384131 2024-12-03T09:16:26,845 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e82b08caef46427cb6158098b48a83d2, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733217384131 2024-12-03T09:16:26,845 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 521efd4a88f04e01a6821463a9991b98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733217384172 2024-12-03T09:16:26,846 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d2aa9ea525465ea56c97a094568b7a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733217384172 2024-12-03T09:16:26,847 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 63e50e96bdc3478d8c730b87c05c8618, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733217384818 2024-12-03T09:16:26,847 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cda3555ebd64c4a9bff089d7fe701f8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733217384818 2024-12-03T09:16:26,874 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#B#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:26,874 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/2dd82073a2d743b68cd836cfc554ee95 is 50, key is test_row_0/B:col10/1733217384824/Put/seqid=0 2024-12-03T09:16:26,885 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:26,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T09:16:26,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:26,902 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T09:16:26,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:26,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:26,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:26,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:26,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:26,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:26,909 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412037f5f23fa3a094b438bde9f65491b5d3f_750235cb928bf724493a7da0f57c7eb3 store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:26,910 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412037f5f23fa3a094b438bde9f65491b5d3f_750235cb928bf724493a7da0f57c7eb3, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:26,910 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037f5f23fa3a094b438bde9f65491b5d3f_750235cb928bf724493a7da0f57c7eb3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:26,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742344_1520 (size=12104) 2024-12-03T09:16:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T09:16:26,920 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/2dd82073a2d743b68cd836cfc554ee95 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/2dd82073a2d743b68cd836cfc554ee95 2024-12-03T09:16:26,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034f418804a3934cf897d04f2b45b3ca3a_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217385983/Put/seqid=0 2024-12-03T09:16:26,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742345_1521 (size=4469) 2024-12-03T09:16:26,924 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#A#compaction#441 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:26,925 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/B of 750235cb928bf724493a7da0f57c7eb3 into 2dd82073a2d743b68cd836cfc554ee95(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:26,925 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:26,925 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/B, priority=13, startTime=1733217386842; duration=0sec 2024-12-03T09:16:26,925 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/8d46ca390a1f467b9b15f435c672e6fb is 175, key is test_row_0/A:col10/1733217384824/Put/seqid=0 2024-12-03T09:16:26,925 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:26,925 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:B 2024-12-03T09:16:26,925 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:26,926 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:26,926 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/C is initiating minor compaction (all files) 2024-12-03T09:16:26,926 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/C in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:26,927 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/f3fd8e73e3624aaba310e6f2b93ba958, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/ff08a28d9388422bb7b8178084d39faf, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/8f2b1b0f688c43eca63311f45d025341] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=32.9 K 2024-12-03T09:16:26,927 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f3fd8e73e3624aaba310e6f2b93ba958, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733217384131 2024-12-03T09:16:26,927 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting ff08a28d9388422bb7b8178084d39faf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733217384172 2024-12-03T09:16:26,927 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f2b1b0f688c43eca63311f45d025341, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733217384818 2024-12-03T09:16:26,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742346_1522 (size=12154) 2024-12-03T09:16:26,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:26,933 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034f418804a3934cf897d04f2b45b3ca3a_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034f418804a3934cf897d04f2b45b3ca3a_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:26,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/baaa7cb31d514c158f3395edacc0e9dd, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:26,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/baaa7cb31d514c158f3395edacc0e9dd is 175, key is test_row_0/A:col10/1733217385983/Put/seqid=0 2024-12-03T09:16:26,939 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:26,940 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/2800f5e13f7d438aa88e03a7c12da9fa is 50, key is test_row_0/C:col10/1733217384824/Put/seqid=0 2024-12-03T09:16:26,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742347_1523 (size=31058) 2024-12-03T09:16:26,946 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/8d46ca390a1f467b9b15f435c672e6fb as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/8d46ca390a1f467b9b15f435c672e6fb 2024-12-03T09:16:26,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742348_1524 (size=30955) 2024-12-03T09:16:26,952 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/A of 750235cb928bf724493a7da0f57c7eb3 into 8d46ca390a1f467b9b15f435c672e6fb(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:26,952 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/baaa7cb31d514c158f3395edacc0e9dd 2024-12-03T09:16:26,952 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:26,952 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/A, priority=13, startTime=1733217386841; duration=0sec 2024-12-03T09:16:26,952 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:26,952 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:A 2024-12-03T09:16:26,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742349_1525 (size=12104) 2024-12-03T09:16:26,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/55a691f9748240d38abf260dbdab75e6 is 50, key is test_row_0/B:col10/1733217385983/Put/seqid=0 2024-12-03T09:16:26,963 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/2800f5e13f7d438aa88e03a7c12da9fa as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/2800f5e13f7d438aa88e03a7c12da9fa 2024-12-03T09:16:26,970 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/C of 750235cb928bf724493a7da0f57c7eb3 into 2800f5e13f7d438aa88e03a7c12da9fa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:26,970 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:26,970 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/C, priority=13, startTime=1733217386842; duration=0sec 2024-12-03T09:16:26,970 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:26,970 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:C 2024-12-03T09:16:26,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742350_1526 (size=12001) 2024-12-03T09:16:26,974 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/55a691f9748240d38abf260dbdab75e6 2024-12-03T09:16:26,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/d7211b9021734016a749fe337396983c is 50, key is test_row_0/C:col10/1733217385983/Put/seqid=0 2024-12-03T09:16:27,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742351_1527 (size=12001) 2024-12-03T09:16:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:27,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:27,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217447152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217447152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217447152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217447153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217447153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217447259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217447259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217447259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217447259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217447259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,409 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/d7211b9021734016a749fe337396983c 2024-12-03T09:16:27,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/baaa7cb31d514c158f3395edacc0e9dd as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/baaa7cb31d514c158f3395edacc0e9dd 2024-12-03T09:16:27,416 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/baaa7cb31d514c158f3395edacc0e9dd, entries=150, sequenceid=84, filesize=30.2 K 2024-12-03T09:16:27,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/55a691f9748240d38abf260dbdab75e6 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/55a691f9748240d38abf260dbdab75e6 2024-12-03T09:16:27,420 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/55a691f9748240d38abf260dbdab75e6, entries=150, sequenceid=84, filesize=11.7 K 2024-12-03T09:16:27,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/d7211b9021734016a749fe337396983c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/d7211b9021734016a749fe337396983c 2024-12-03T09:16:27,423 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/d7211b9021734016a749fe337396983c, entries=150, sequenceid=84, filesize=11.7 K 2024-12-03T09:16:27,423 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 750235cb928bf724493a7da0f57c7eb3 in 521ms, sequenceid=84, compaction requested=false 2024-12-03T09:16:27,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:27,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:27,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-03T09:16:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-03T09:16:27,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-03T09:16:27,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1330 sec 2024-12-03T09:16:27,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.1370 sec 2024-12-03T09:16:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T09:16:27,430 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:27,430 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-03T09:16:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T09:16:27,431 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:27,432 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:27,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:27,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T09:16:27,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:27,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:27,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:27,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:27,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:27,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:27,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203adb879242cec4914b802c43c4d97b78d_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217387126/Put/seqid=0 2024-12-03T09:16:27,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742352_1528 (size=12154) 2024-12-03T09:16:27,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217447492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217447494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217447496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217447496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217447496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T09:16:27,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T09:16:27,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:27,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:27,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:27,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:27,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:27,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217447599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217447610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217447610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217447610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217447610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T09:16:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:27,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T09:16:27,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217447810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217447819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217447819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217447819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217447819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:27,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:27,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T09:16:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:27,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:27,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:27,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:27,892 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:27,895 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203adb879242cec4914b802c43c4d97b78d_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203adb879242cec4914b802c43c4d97b78d_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:27,895 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/893666939fba46569441f9aca9e55284, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:27,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/893666939fba46569441f9aca9e55284 is 175, key is test_row_0/A:col10/1733217387126/Put/seqid=0 2024-12-03T09:16:27,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742353_1529 (size=30955) 2024-12-03T09:16:28,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T09:16:28,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:28,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T09:16:28,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217448119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:28,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:28,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217448129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:28,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217448129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:28,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217448129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:28,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217448129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:28,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T09:16:28,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:28,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,299 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/893666939fba46569441f9aca9e55284 2024-12-03T09:16:28,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/6a1bdb8f430148d5912394c6a84dcf34 is 50, key is test_row_0/B:col10/1733217387126/Put/seqid=0 2024-12-03T09:16:28,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742354_1530 (size=12001) 2024-12-03T09:16:28,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T09:16:28,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:28,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T09:16:28,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:28,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T09:16:28,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217448629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:28,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:28,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217448639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:28,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:28,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217448639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:28,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:28,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217448639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217448639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:28,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:28,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T09:16:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/6a1bdb8f430148d5912394c6a84dcf34 2024-12-03T09:16:28,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/76ed6a14fbb547c49d698f3390a05592 is 50, key is test_row_0/C:col10/1733217387126/Put/seqid=0 2024-12-03T09:16:28,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742355_1531 (size=12001) 2024-12-03T09:16:28,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T09:16:28,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:28,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T09:16:28,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:28,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:28,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:29,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T09:16:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:29,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:29,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/76ed6a14fbb547c49d698f3390a05592 2024-12-03T09:16:29,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/893666939fba46569441f9aca9e55284 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/893666939fba46569441f9aca9e55284 2024-12-03T09:16:29,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/893666939fba46569441f9aca9e55284, entries=150, sequenceid=103, filesize=30.2 K 2024-12-03T09:16:29,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/6a1bdb8f430148d5912394c6a84dcf34 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/6a1bdb8f430148d5912394c6a84dcf34 2024-12-03T09:16:29,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/6a1bdb8f430148d5912394c6a84dcf34, entries=150, sequenceid=103, filesize=11.7 K 2024-12-03T09:16:29,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/76ed6a14fbb547c49d698f3390a05592 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/76ed6a14fbb547c49d698f3390a05592 2024-12-03T09:16:29,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/76ed6a14fbb547c49d698f3390a05592, entries=150, sequenceid=103, filesize=11.7 K 2024-12-03T09:16:29,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 750235cb928bf724493a7da0f57c7eb3 in 1657ms, sequenceid=103, compaction requested=true 2024-12-03T09:16:29,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:29,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:29,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:29,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:29,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:29,129 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:29,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:29,129 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:29,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:29,130 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:29,130 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:29,130 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/A is initiating minor compaction (all files) 2024-12-03T09:16:29,130 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/B is initiating minor compaction (all files) 2024-12-03T09:16:29,130 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/A in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:29,130 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/8d46ca390a1f467b9b15f435c672e6fb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/baaa7cb31d514c158f3395edacc0e9dd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/893666939fba46569441f9aca9e55284] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=90.8 K 2024-12-03T09:16:29,130 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:29,130 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/8d46ca390a1f467b9b15f435c672e6fb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/baaa7cb31d514c158f3395edacc0e9dd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/893666939fba46569441f9aca9e55284] 2024-12-03T09:16:29,130 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/B in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:29,130 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/2dd82073a2d743b68cd836cfc554ee95, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/55a691f9748240d38abf260dbdab75e6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/6a1bdb8f430148d5912394c6a84dcf34] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=35.3 K 2024-12-03T09:16:29,130 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d46ca390a1f467b9b15f435c672e6fb, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733217384818 2024-12-03T09:16:29,130 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dd82073a2d743b68cd836cfc554ee95, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733217384818 2024-12-03T09:16:29,131 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting baaa7cb31d514c158f3395edacc0e9dd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733217385982 2024-12-03T09:16:29,131 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 55a691f9748240d38abf260dbdab75e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733217385982 2024-12-03T09:16:29,131 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 893666939fba46569441f9aca9e55284, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733217387126 2024-12-03T09:16:29,131 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a1bdb8f430148d5912394c6a84dcf34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733217387126 2024-12-03T09:16:29,134 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:29,136 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#B#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:29,136 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/ac0d250a95654826916610a7b911d87a is 50, key is test_row_0/B:col10/1733217387126/Put/seqid=0 2024-12-03T09:16:29,138 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203f17db680b29848568fe51689f43fc681_750235cb928bf724493a7da0f57c7eb3 store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:29,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742356_1532 (size=12207) 2024-12-03T09:16:29,139 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203f17db680b29848568fe51689f43fc681_750235cb928bf724493a7da0f57c7eb3, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:29,139 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f17db680b29848568fe51689f43fc681_750235cb928bf724493a7da0f57c7eb3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:29,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742357_1533 (size=4469) 2024-12-03T09:16:29,143 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/ac0d250a95654826916610a7b911d87a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/ac0d250a95654826916610a7b911d87a 2024-12-03T09:16:29,147 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/B of 750235cb928bf724493a7da0f57c7eb3 into ac0d250a95654826916610a7b911d87a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:29,147 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:29,147 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/B, priority=13, startTime=1733217389129; duration=0sec 2024-12-03T09:16:29,147 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:29,147 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:B 2024-12-03T09:16:29,147 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:29,148 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:29,148 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/C is initiating minor compaction (all files) 2024-12-03T09:16:29,148 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/C in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:29,148 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/2800f5e13f7d438aa88e03a7c12da9fa, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/d7211b9021734016a749fe337396983c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/76ed6a14fbb547c49d698f3390a05592] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=35.3 K 2024-12-03T09:16:29,148 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2800f5e13f7d438aa88e03a7c12da9fa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733217384818 2024-12-03T09:16:29,148 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting d7211b9021734016a749fe337396983c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733217385982 2024-12-03T09:16:29,148 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 76ed6a14fbb547c49d698f3390a05592, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733217387126 2024-12-03T09:16:29,154 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#C#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:29,155 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/6145b19e879146d889f22498a56573a4 is 50, key is test_row_0/C:col10/1733217387126/Put/seqid=0 2024-12-03T09:16:29,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742358_1534 (size=12207) 2024-12-03T09:16:29,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T09:16:29,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:29,261 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T09:16:29,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:29,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:29,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:29,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:29,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:29,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:29,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203abb1fb4e42984b589eedb476ff52e04e_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217387495/Put/seqid=0 2024-12-03T09:16:29,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742359_1535 (size=12154) 2024-12-03T09:16:29,544 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#A#compaction#450 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:29,545 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/bcd9a2c850ac40de92b878fbb0c0ab3a is 175, key is test_row_0/A:col10/1733217387126/Put/seqid=0 2024-12-03T09:16:29,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742360_1536 (size=31161) 2024-12-03T09:16:29,551 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/bcd9a2c850ac40de92b878fbb0c0ab3a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/bcd9a2c850ac40de92b878fbb0c0ab3a 2024-12-03T09:16:29,554 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/A of 750235cb928bf724493a7da0f57c7eb3 into bcd9a2c850ac40de92b878fbb0c0ab3a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:29,554 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:29,554 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/A, priority=13, startTime=1733217389129; duration=0sec 2024-12-03T09:16:29,554 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:29,554 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:A 2024-12-03T09:16:29,561 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/6145b19e879146d889f22498a56573a4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6145b19e879146d889f22498a56573a4 2024-12-03T09:16:29,564 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/C of 750235cb928bf724493a7da0f57c7eb3 into 6145b19e879146d889f22498a56573a4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:29,564 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:29,564 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/C, priority=13, startTime=1733217389129; duration=0sec 2024-12-03T09:16:29,564 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:29,564 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:C 2024-12-03T09:16:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T09:16:29,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:29,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217449655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:29,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217449659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:29,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:29,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217449659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:29,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:29,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217449659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:29,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217449659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:29,662 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:29,662 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:29,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:29,673 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203abb1fb4e42984b589eedb476ff52e04e_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203abb1fb4e42984b589eedb476ff52e04e_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:29,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/5c753c98913c46c18f180a545de3273e, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:29,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/5c753c98913c46c18f180a545de3273e is 175, key is test_row_0/A:col10/1733217387495/Put/seqid=0 2024-12-03T09:16:29,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742361_1537 (size=30955) 2024-12-03T09:16:29,682 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/5c753c98913c46c18f180a545de3273e 2024-12-03T09:16:29,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/c9d277a98ef34a75be255b53a4165341 is 50, key is test_row_0/B:col10/1733217387495/Put/seqid=0 2024-12-03T09:16:29,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742362_1538 (size=12001) 2024-12-03T09:16:29,696 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/c9d277a98ef34a75be255b53a4165341 2024-12-03T09:16:29,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/9aa8f893da484dc39635b0cb0edf3485 is 50, key is test_row_0/C:col10/1733217387495/Put/seqid=0 2024-12-03T09:16:29,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742363_1539 (size=12001) 2024-12-03T09:16:29,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217449769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:29,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217449769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217449769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:29,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217449979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:29,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:29,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217449979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:29,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217449979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:29,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:29,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:29,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:29,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:30,108 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/9aa8f893da484dc39635b0cb0edf3485 2024-12-03T09:16:30,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/5c753c98913c46c18f180a545de3273e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/5c753c98913c46c18f180a545de3273e 2024-12-03T09:16:30,114 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/5c753c98913c46c18f180a545de3273e, entries=150, sequenceid=121, filesize=30.2 K 2024-12-03T09:16:30,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/c9d277a98ef34a75be255b53a4165341 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/c9d277a98ef34a75be255b53a4165341 2024-12-03T09:16:30,117 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/c9d277a98ef34a75be255b53a4165341, entries=150, sequenceid=121, filesize=11.7 K 2024-12-03T09:16:30,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/9aa8f893da484dc39635b0cb0edf3485 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/9aa8f893da484dc39635b0cb0edf3485 2024-12-03T09:16:30,120 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/9aa8f893da484dc39635b0cb0edf3485, entries=150, sequenceid=121, filesize=11.7 K 2024-12-03T09:16:30,121 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 750235cb928bf724493a7da0f57c7eb3 in 861ms, sequenceid=121, compaction requested=false 2024-12-03T09:16:30,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:30,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:30,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-03T09:16:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-03T09:16:30,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-03T09:16:30,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6900 sec 2024-12-03T09:16:30,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.6940 sec 2024-12-03T09:16:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:30,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T09:16:30,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:30,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:30,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:30,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:30,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:30,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:30,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120315fd638e4c0d484b8d4e7389c9ac7af4_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217389657/Put/seqid=0 2024-12-03T09:16:30,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742364_1540 (size=12304) 2024-12-03T09:16:30,308 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:30,312 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120315fd638e4c0d484b8d4e7389c9ac7af4_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120315fd638e4c0d484b8d4e7389c9ac7af4_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:30,313 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/f8cba1dc883e4cb58164a94a8dae779d, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:30,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/f8cba1dc883e4cb58164a94a8dae779d is 175, key is test_row_0/A:col10/1733217389657/Put/seqid=0 2024-12-03T09:16:30,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217450310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:30,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217450313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:30,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217450313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:30,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:30,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742365_1541 (size=31105) 2024-12-03T09:16:30,328 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/f8cba1dc883e4cb58164a94a8dae779d 2024-12-03T09:16:30,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/7082da66a42942568cb489aee76f8521 is 50, key is test_row_0/B:col10/1733217389657/Put/seqid=0 2024-12-03T09:16:30,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742366_1542 (size=12151) 2024-12-03T09:16:30,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/7082da66a42942568cb489aee76f8521 2024-12-03T09:16:30,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/f3da1a78ba37476c936e05f0d549977a is 50, key is test_row_0/C:col10/1733217389657/Put/seqid=0 2024-12-03T09:16:30,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742367_1543 (size=12151) 2024-12-03T09:16:30,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/f3da1a78ba37476c936e05f0d549977a 2024-12-03T09:16:30,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/f8cba1dc883e4cb58164a94a8dae779d as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f8cba1dc883e4cb58164a94a8dae779d 2024-12-03T09:16:30,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f8cba1dc883e4cb58164a94a8dae779d, entries=150, sequenceid=139, filesize=30.4 K 2024-12-03T09:16:30,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/7082da66a42942568cb489aee76f8521 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/7082da66a42942568cb489aee76f8521 2024-12-03T09:16:30,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/7082da66a42942568cb489aee76f8521, entries=150, sequenceid=139, filesize=11.9 K 2024-12-03T09:16:30,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/f3da1a78ba37476c936e05f0d549977a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/f3da1a78ba37476c936e05f0d549977a 2024-12-03T09:16:30,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/f3da1a78ba37476c936e05f0d549977a, entries=150, sequenceid=139, filesize=11.9 K 2024-12-03T09:16:30,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 750235cb928bf724493a7da0f57c7eb3 in 83ms, sequenceid=139, compaction requested=true 2024-12-03T09:16:30,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:30,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:30,374 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:30,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:30,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:30,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:30,374 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:30,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:30,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:30,375 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:30,375 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:30,375 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/A is initiating minor compaction (all files) 2024-12-03T09:16:30,375 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/B is initiating minor compaction (all files) 2024-12-03T09:16:30,375 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/A in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:30,375 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/B in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:30,375 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/bcd9a2c850ac40de92b878fbb0c0ab3a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/5c753c98913c46c18f180a545de3273e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f8cba1dc883e4cb58164a94a8dae779d] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=91.0 K 2024-12-03T09:16:30,375 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/ac0d250a95654826916610a7b911d87a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/c9d277a98ef34a75be255b53a4165341, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/7082da66a42942568cb489aee76f8521] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=35.5 K 2024-12-03T09:16:30,375 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:30,375 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/bcd9a2c850ac40de92b878fbb0c0ab3a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/5c753c98913c46c18f180a545de3273e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f8cba1dc883e4cb58164a94a8dae779d] 2024-12-03T09:16:30,376 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcd9a2c850ac40de92b878fbb0c0ab3a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733217387126 2024-12-03T09:16:30,376 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting ac0d250a95654826916610a7b911d87a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733217387126 2024-12-03T09:16:30,376 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c753c98913c46c18f180a545de3273e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733217387487 2024-12-03T09:16:30,376 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting c9d277a98ef34a75be255b53a4165341, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733217387487 2024-12-03T09:16:30,377 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8cba1dc883e4cb58164a94a8dae779d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733217389657 2024-12-03T09:16:30,377 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 7082da66a42942568cb489aee76f8521, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733217389657 2024-12-03T09:16:30,382 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:30,383 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#B#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:30,383 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/bdae2a15aea94720a6cdeb40379b84cd is 50, key is test_row_0/B:col10/1733217389657/Put/seqid=0 2024-12-03T09:16:30,385 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203a7e92f099ae84174b4bd5d267cacc10b_750235cb928bf724493a7da0f57c7eb3 store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:30,387 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203a7e92f099ae84174b4bd5d267cacc10b_750235cb928bf724493a7da0f57c7eb3, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:30,387 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a7e92f099ae84174b4bd5d267cacc10b_750235cb928bf724493a7da0f57c7eb3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:30,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742369_1545 (size=4469) 2024-12-03T09:16:30,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742368_1544 (size=12459) 2024-12-03T09:16:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:30,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T09:16:30,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:30,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:30,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:30,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:30,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:30,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:30,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032667a0e5b7f941c2b27fc47ed1cfe983_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217390312/Put/seqid=0 2024-12-03T09:16:30,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742370_1546 (size=17284) 2024-12-03T09:16:30,438 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:30,441 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032667a0e5b7f941c2b27fc47ed1cfe983_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032667a0e5b7f941c2b27fc47ed1cfe983_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:30,443 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/ca8de862f72443beba56183fde0cfdad, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:30,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/ca8de862f72443beba56183fde0cfdad is 175, key is test_row_0/A:col10/1733217390312/Put/seqid=0 2024-12-03T09:16:30,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217450441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:30,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217450442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:30,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217450443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:30,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:30,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742371_1547 (size=48389) 2024-12-03T09:16:30,450 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/ca8de862f72443beba56183fde0cfdad 2024-12-03T09:16:30,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/4b054c1e578e4009ae85cc4dec42432e is 50, key is test_row_0/B:col10/1733217390312/Put/seqid=0 2024-12-03T09:16:30,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742372_1548 (size=12151) 2024-12-03T09:16:30,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217450550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:30,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217450550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:30,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217450550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:30,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217450759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:30,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217450759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:30,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217450759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:30,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:30,793 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#A#compaction#459 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:30,794 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/4b8dacad9c49434ba7689a481985a765 is 175, key is test_row_0/A:col10/1733217389657/Put/seqid=0 2024-12-03T09:16:30,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742373_1549 (size=31413) 2024-12-03T09:16:30,804 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/bdae2a15aea94720a6cdeb40379b84cd as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/bdae2a15aea94720a6cdeb40379b84cd 2024-12-03T09:16:30,808 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/B of 750235cb928bf724493a7da0f57c7eb3 into bdae2a15aea94720a6cdeb40379b84cd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:30,808 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:30,808 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/B, priority=13, startTime=1733217390374; duration=0sec 2024-12-03T09:16:30,808 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:30,808 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:B 2024-12-03T09:16:30,808 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:30,809 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:30,809 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/C is initiating minor compaction (all files) 2024-12-03T09:16:30,809 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/C in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:30,809 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6145b19e879146d889f22498a56573a4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/9aa8f893da484dc39635b0cb0edf3485, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/f3da1a78ba37476c936e05f0d549977a] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=35.5 K 2024-12-03T09:16:30,809 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6145b19e879146d889f22498a56573a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733217387126 2024-12-03T09:16:30,809 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aa8f893da484dc39635b0cb0edf3485, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733217387487 2024-12-03T09:16:30,810 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting f3da1a78ba37476c936e05f0d549977a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733217389657 2024-12-03T09:16:30,815 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:30,816 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/663d6e1af3df472a9e2261512ed94c87 is 50, key is test_row_0/C:col10/1733217389657/Put/seqid=0 2024-12-03T09:16:30,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742374_1550 (size=12459) 2024-12-03T09:16:30,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/4b054c1e578e4009ae85cc4dec42432e 2024-12-03T09:16:30,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/6ba81c8484f74246ab6f62bdf8175347 is 50, key is test_row_0/C:col10/1733217390312/Put/seqid=0 2024-12-03T09:16:30,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742375_1551 (size=12151) 2024-12-03T09:16:31,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217451069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217451069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217451069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,200 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/4b8dacad9c49434ba7689a481985a765 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4b8dacad9c49434ba7689a481985a765 2024-12-03T09:16:31,204 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/A of 750235cb928bf724493a7da0f57c7eb3 into 4b8dacad9c49434ba7689a481985a765(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:31,204 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:31,204 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/A, priority=13, startTime=1733217390374; duration=0sec 2024-12-03T09:16:31,204 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:31,204 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:A 2024-12-03T09:16:31,223 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/663d6e1af3df472a9e2261512ed94c87 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/663d6e1af3df472a9e2261512ed94c87 2024-12-03T09:16:31,227 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/C of 750235cb928bf724493a7da0f57c7eb3 into 663d6e1af3df472a9e2261512ed94c87(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:31,227 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:31,227 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/C, priority=13, startTime=1733217390374; duration=0sec 2024-12-03T09:16:31,227 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:31,227 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:C 2024-12-03T09:16:31,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/6ba81c8484f74246ab6f62bdf8175347 2024-12-03T09:16:31,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/ca8de862f72443beba56183fde0cfdad as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/ca8de862f72443beba56183fde0cfdad 2024-12-03T09:16:31,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/ca8de862f72443beba56183fde0cfdad, entries=250, sequenceid=157, filesize=47.3 K 2024-12-03T09:16:31,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/4b054c1e578e4009ae85cc4dec42432e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/4b054c1e578e4009ae85cc4dec42432e 2024-12-03T09:16:31,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/4b054c1e578e4009ae85cc4dec42432e, entries=150, sequenceid=157, filesize=11.9 K 2024-12-03T09:16:31,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/6ba81c8484f74246ab6f62bdf8175347 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6ba81c8484f74246ab6f62bdf8175347 2024-12-03T09:16:31,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6ba81c8484f74246ab6f62bdf8175347, entries=150, sequenceid=157, filesize=11.9 K 2024-12-03T09:16:31,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 750235cb928bf724493a7da0f57c7eb3 in 869ms, sequenceid=157, compaction requested=false 2024-12-03T09:16:31,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:31,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T09:16:31,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:31,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:31,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:31,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:31,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:31,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:31,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120346fc2e301b3d4c6bbf44a6eaa7a7d745_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217390439/Put/seqid=0 2024-12-03T09:16:31,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742376_1552 (size=14794) 2024-12-03T09:16:31,598 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:31,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T09:16:31,599 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:31,600 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-03T09:16:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T09:16:31,602 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:31,602 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:31,602 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120346fc2e301b3d4c6bbf44a6eaa7a7d745_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120346fc2e301b3d4c6bbf44a6eaa7a7d745_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:31,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:31,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,604 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/4c0147242a854935baa5445cacbc2c6b, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217451600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/4c0147242a854935baa5445cacbc2c6b is 175, key is test_row_0/A:col10/1733217390439/Put/seqid=0 2024-12-03T09:16:31,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217451602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742377_1553 (size=39749) 2024-12-03T09:16:31,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217451604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,608 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/4c0147242a854935baa5445cacbc2c6b 2024-12-03T09:16:31,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/774d1aa18e514bba911564a2a6ab4773 is 50, key is test_row_0/B:col10/1733217390439/Put/seqid=0 2024-12-03T09:16:31,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742378_1554 (size=12151) 2024-12-03T09:16:31,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/774d1aa18e514bba911564a2a6ab4773 2024-12-03T09:16:31,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/22c8d563137f4a388f6729880d93d18e is 50, key is test_row_0/C:col10/1733217390439/Put/seqid=0 2024-12-03T09:16:31,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742379_1555 (size=12151) 2024-12-03T09:16:31,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/22c8d563137f4a388f6729880d93d18e 2024-12-03T09:16:31,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/4c0147242a854935baa5445cacbc2c6b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4c0147242a854935baa5445cacbc2c6b 2024-12-03T09:16:31,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4c0147242a854935baa5445cacbc2c6b, entries=200, sequenceid=177, filesize=38.8 K 2024-12-03T09:16:31,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/774d1aa18e514bba911564a2a6ab4773 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/774d1aa18e514bba911564a2a6ab4773 2024-12-03T09:16:31,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/774d1aa18e514bba911564a2a6ab4773, entries=150, sequenceid=177, filesize=11.9 K 2024-12-03T09:16:31,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/22c8d563137f4a388f6729880d93d18e as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/22c8d563137f4a388f6729880d93d18e 2024-12-03T09:16:31,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/22c8d563137f4a388f6729880d93d18e, entries=150, sequenceid=177, filesize=11.9 K 2024-12-03T09:16:31,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 750235cb928bf724493a7da0f57c7eb3 in 59ms, sequenceid=177, compaction requested=true 2024-12-03T09:16:31,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:31,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:31,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:31,640 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:31,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:31,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:31,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:31,640 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:31,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:31,641 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:31,641 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/A is initiating minor compaction (all files) 2024-12-03T09:16:31,641 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/A in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:31,641 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4b8dacad9c49434ba7689a481985a765, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/ca8de862f72443beba56183fde0cfdad, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4c0147242a854935baa5445cacbc2c6b] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=116.7 K 2024-12-03T09:16:31,641 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:31,641 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4b8dacad9c49434ba7689a481985a765, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/ca8de862f72443beba56183fde0cfdad, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4c0147242a854935baa5445cacbc2c6b] 2024-12-03T09:16:31,641 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:31,641 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/B is initiating minor compaction (all files) 2024-12-03T09:16:31,641 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/B in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:31,641 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/bdae2a15aea94720a6cdeb40379b84cd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/4b054c1e578e4009ae85cc4dec42432e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/774d1aa18e514bba911564a2a6ab4773] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=35.9 K 2024-12-03T09:16:31,641 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b8dacad9c49434ba7689a481985a765, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733217389657 2024-12-03T09:16:31,643 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting bdae2a15aea94720a6cdeb40379b84cd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733217389657 2024-12-03T09:16:31,643 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca8de862f72443beba56183fde0cfdad, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733217390302 2024-12-03T09:16:31,643 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b054c1e578e4009ae85cc4dec42432e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733217390312 2024-12-03T09:16:31,643 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c0147242a854935baa5445cacbc2c6b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733217390439 2024-12-03T09:16:31,643 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 774d1aa18e514bba911564a2a6ab4773, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733217390439 2024-12-03T09:16:31,649 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:31,654 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:31,655 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/30ad4c17b8c04a4c837fc57ef75c1f3b is 50, key is test_row_0/B:col10/1733217390439/Put/seqid=0 2024-12-03T09:16:31,661 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412033005eb701ff5458daad4aaf054997a02_750235cb928bf724493a7da0f57c7eb3 store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:31,662 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412033005eb701ff5458daad4aaf054997a02_750235cb928bf724493a7da0f57c7eb3, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:31,662 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033005eb701ff5458daad4aaf054997a02_750235cb928bf724493a7da0f57c7eb3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:31,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742380_1556 (size=12561) 2024-12-03T09:16:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:31,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T09:16:31,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:31,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:31,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:31,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:31,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:31,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:31,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742381_1557 (size=4469) 2024-12-03T09:16:31,676 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#A#compaction#467 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:31,676 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/f81972fea845425da7b944af46229e11 is 175, key is test_row_0/A:col10/1733217390439/Put/seqid=0 2024-12-03T09:16:31,676 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/30ad4c17b8c04a4c837fc57ef75c1f3b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/30ad4c17b8c04a4c837fc57ef75c1f3b 2024-12-03T09:16:31,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120330e1dc83eb3b4873a4dee8462b6045cf_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217391600/Put/seqid=0 2024-12-03T09:16:31,682 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/B of 750235cb928bf724493a7da0f57c7eb3 into 30ad4c17b8c04a4c837fc57ef75c1f3b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:31,682 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:31,682 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/B, priority=13, startTime=1733217391640; duration=0sec 2024-12-03T09:16:31,682 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:31,682 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:B 2024-12-03T09:16:31,682 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:31,683 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:31,683 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/C is initiating minor compaction (all files) 2024-12-03T09:16:31,683 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/C in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:31,683 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/663d6e1af3df472a9e2261512ed94c87, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6ba81c8484f74246ab6f62bdf8175347, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/22c8d563137f4a388f6729880d93d18e] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=35.9 K 2024-12-03T09:16:31,683 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 663d6e1af3df472a9e2261512ed94c87, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1733217389657 2024-12-03T09:16:31,684 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ba81c8484f74246ab6f62bdf8175347, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733217390312 2024-12-03T09:16:31,684 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 22c8d563137f4a388f6729880d93d18e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733217390439 2024-12-03T09:16:31,694 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:31,694 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/e1215d9d93a44663b861d61b6cc8a344 is 50, key is test_row_0/C:col10/1733217390439/Put/seqid=0 2024-12-03T09:16:31,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742382_1558 (size=31515) 2024-12-03T09:16:31,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217451701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742383_1559 (size=14794) 2024-12-03T09:16:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742384_1560 (size=12561) 2024-12-03T09:16:31,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217451704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T09:16:31,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217451709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217451709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217451709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T09:16:31,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:31,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:31,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:31,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:31,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:31,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217451809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217451809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T09:16:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:31,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:31,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T09:16:31,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217451919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217451919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:31,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217451919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:32,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217452019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:32,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217452019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:32,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T09:16:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:32,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,101 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/f81972fea845425da7b944af46229e11 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f81972fea845425da7b944af46229e11 2024-12-03T09:16:32,104 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/A of 750235cb928bf724493a7da0f57c7eb3 into f81972fea845425da7b944af46229e11(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:32,104 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:32,104 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:32,104 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/A, priority=13, startTime=1733217391640; duration=0sec 2024-12-03T09:16:32,104 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:32,104 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:A 2024-12-03T09:16:32,110 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120330e1dc83eb3b4873a4dee8462b6045cf_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120330e1dc83eb3b4873a4dee8462b6045cf_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:32,110 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/05d502f3616049ea8a2b8a1a7f23dafc, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:32,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/05d502f3616049ea8a2b8a1a7f23dafc is 175, key is test_row_0/A:col10/1733217391600/Put/seqid=0 2024-12-03T09:16:32,112 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/e1215d9d93a44663b861d61b6cc8a344 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/e1215d9d93a44663b861d61b6cc8a344 2024-12-03T09:16:32,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742385_1561 (size=39749) 2024-12-03T09:16:32,117 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/C of 750235cb928bf724493a7da0f57c7eb3 into e1215d9d93a44663b861d61b6cc8a344(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:32,117 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:32,117 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/C, priority=13, startTime=1733217391640; duration=0sec 2024-12-03T09:16:32,117 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:32,117 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:C 2024-12-03T09:16:32,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T09:16:32,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:32,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T09:16:32,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217452229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:32,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217452229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:32,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217452229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:32,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217452329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:32,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217452329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:32,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T09:16:32,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:32,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,515 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/05d502f3616049ea8a2b8a1a7f23dafc 2024-12-03T09:16:32,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T09:16:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:32,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/353598b1c2fa40818a93db6066a0c4e7 is 50, key is test_row_0/B:col10/1733217391600/Put/seqid=0 2024-12-03T09:16:32,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742386_1562 (size=12151) 2024-12-03T09:16:32,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T09:16:32,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T09:16:32,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:32,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217452739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217452739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:32,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217452740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:32,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T09:16:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217452839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:32,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:32,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217452840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:32,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:32,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:32,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:32,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/353598b1c2fa40818a93db6066a0c4e7 2024-12-03T09:16:32,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/3a1790c3ec364cec9ff6968831ace777 is 50, key is test_row_0/C:col10/1733217391600/Put/seqid=0 2024-12-03T09:16:32,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742387_1563 (size=12151) 2024-12-03T09:16:32,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T09:16:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:32,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:33,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T09:16:33,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:33,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:33,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:33,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:33,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T09:16:33,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:33,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:33,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:33,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:33,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:33,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/3a1790c3ec364cec9ff6968831ace777 2024-12-03T09:16:33,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/05d502f3616049ea8a2b8a1a7f23dafc as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/05d502f3616049ea8a2b8a1a7f23dafc 2024-12-03T09:16:33,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/05d502f3616049ea8a2b8a1a7f23dafc, entries=200, sequenceid=193, filesize=38.8 K 2024-12-03T09:16:33,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/353598b1c2fa40818a93db6066a0c4e7 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/353598b1c2fa40818a93db6066a0c4e7 2024-12-03T09:16:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/353598b1c2fa40818a93db6066a0c4e7, entries=150, sequenceid=193, filesize=11.9 K 2024-12-03T09:16:33,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/3a1790c3ec364cec9ff6968831ace777 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/3a1790c3ec364cec9ff6968831ace777 2024-12-03T09:16:33,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/3a1790c3ec364cec9ff6968831ace777, entries=150, sequenceid=193, filesize=11.9 K 2024-12-03T09:16:33,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 750235cb928bf724493a7da0f57c7eb3 in 1681ms, sequenceid=193, compaction requested=false 2024-12-03T09:16:33,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:33,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T09:16:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:33,433 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T09:16:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:33,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:33,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a30e147c48ff48b29d8ff2a2556f830b_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217391702/Put/seqid=0 2024-12-03T09:16:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742388_1564 (size=9814) 2024-12-03T09:16:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T09:16:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217453784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:33,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217453786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:33,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217453787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:33,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:33,846 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a30e147c48ff48b29d8ff2a2556f830b_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a30e147c48ff48b29d8ff2a2556f830b_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/447eb4b9845c44ec934d871db504ad07, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/447eb4b9845c44ec934d871db504ad07 is 175, key is test_row_0/A:col10/1733217391702/Put/seqid=0 2024-12-03T09:16:33,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217453849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:33,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217453850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:33,852 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:33,852 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:33,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742389_1565 (size=22461) 2024-12-03T09:16:33,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217453899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:33,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217453899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:33,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217453899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:34,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T09:16:34,053 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T09:16:34,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:34,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217454109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:34,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:34,111 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217454109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:34,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217454109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:34,278 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/447eb4b9845c44ec934d871db504ad07 2024-12-03T09:16:34,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/2c702a326d40401e983820a1cd2355b4 is 50, key is test_row_0/B:col10/1733217391702/Put/seqid=0 2024-12-03T09:16:34,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742390_1566 (size=9757) 2024-12-03T09:16:34,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:34,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217454420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217454420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:34,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217454420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:34,687 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/2c702a326d40401e983820a1cd2355b4 2024-12-03T09:16:34,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/556be96e1fa04f28b80fbf41d7adfd5a is 50, key is test_row_0/C:col10/1733217391702/Put/seqid=0 2024-12-03T09:16:34,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742391_1567 (size=9757) 2024-12-03T09:16:34,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217454930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217454930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:34,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217454930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:35,100 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/556be96e1fa04f28b80fbf41d7adfd5a 2024-12-03T09:16:35,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/447eb4b9845c44ec934d871db504ad07 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/447eb4b9845c44ec934d871db504ad07 2024-12-03T09:16:35,106 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/447eb4b9845c44ec934d871db504ad07, entries=100, sequenceid=212, filesize=21.9 K 2024-12-03T09:16:35,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/2c702a326d40401e983820a1cd2355b4 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/2c702a326d40401e983820a1cd2355b4 2024-12-03T09:16:35,110 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/2c702a326d40401e983820a1cd2355b4, entries=100, sequenceid=212, filesize=9.5 K 2024-12-03T09:16:35,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/556be96e1fa04f28b80fbf41d7adfd5a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/556be96e1fa04f28b80fbf41d7adfd5a 2024-12-03T09:16:35,113 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/556be96e1fa04f28b80fbf41d7adfd5a, entries=100, sequenceid=212, filesize=9.5 K 2024-12-03T09:16:35,114 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 750235cb928bf724493a7da0f57c7eb3 in 1681ms, sequenceid=212, compaction requested=true 2024-12-03T09:16:35,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:35,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:35,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-03T09:16:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-03T09:16:35,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-03T09:16:35,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5140 sec 2024-12-03T09:16:35,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.5190 sec 2024-12-03T09:16:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T09:16:35,770 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:35,770 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-03T09:16:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T09:16:35,773 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:35,773 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:35,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:35,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T09:16:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:35,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120372f6411eebe6442a8e26ad5ee3db7a36_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217393781/Put/seqid=0 2024-12-03T09:16:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T09:16:35,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742392_1568 (size=14794) 2024-12-03T09:16:35,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217455895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:35,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:35,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:35,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:35,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217455898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:35,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:35,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:35,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:35,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T09:16:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:35,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:35,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:35,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217455940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:35,942 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:35,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217455949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:35,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217455949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:35,952 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:35,953 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:36,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217456009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:36,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217456009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:36,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T09:16:36,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:36,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T09:16:36,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217456219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:36,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217456219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:36,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T09:16:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,280 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:36,283 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120372f6411eebe6442a8e26ad5ee3db7a36_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120372f6411eebe6442a8e26ad5ee3db7a36_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:36,284 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/e4a016aa41b5479f8790e56dfa48ad97, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:36,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/e4a016aa41b5479f8790e56dfa48ad97 is 175, key is test_row_0/A:col10/1733217393781/Put/seqid=0 2024-12-03T09:16:36,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742393_1569 (size=39749) 2024-12-03T09:16:36,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T09:16:36,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:36,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T09:16:36,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217456530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:36,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:36,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217456530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:36,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T09:16:36,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:36,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,688 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/e4a016aa41b5479f8790e56dfa48ad97 2024-12-03T09:16:36,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T09:16:36,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:36,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/65747ca4fba844bc85b090a20b3537ce is 50, key is test_row_0/B:col10/1733217393781/Put/seqid=0 2024-12-03T09:16:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742394_1570 (size=12151) 2024-12-03T09:16:36,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T09:16:36,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:36,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T09:16:36,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T09:16:36,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:36,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:36,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:37,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:37,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217457039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:37,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:37,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:37,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217457039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:37,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:37,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/65747ca4fba844bc85b090a20b3537ce 2024-12-03T09:16:37,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/fbb2370143744ab881a54d39c33ababa is 50, key is test_row_0/C:col10/1733217393781/Put/seqid=0 2024-12-03T09:16:37,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742395_1571 (size=12151) 2024-12-03T09:16:37,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T09:16:37,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:37,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:37,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:37,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:37,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:37,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:37,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T09:16:37,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:37,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:37,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:37,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:37,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:37,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T09:16:37,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:37,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:37,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:37,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:37,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:37,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/fbb2370143744ab881a54d39c33ababa 2024-12-03T09:16:37,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/e4a016aa41b5479f8790e56dfa48ad97 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e4a016aa41b5479f8790e56dfa48ad97 2024-12-03T09:16:37,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e4a016aa41b5479f8790e56dfa48ad97, entries=200, sequenceid=229, filesize=38.8 K 2024-12-03T09:16:37,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/65747ca4fba844bc85b090a20b3537ce as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/65747ca4fba844bc85b090a20b3537ce 2024-12-03T09:16:37,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/65747ca4fba844bc85b090a20b3537ce, entries=150, sequenceid=229, filesize=11.9 K 2024-12-03T09:16:37,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/fbb2370143744ab881a54d39c33ababa as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/fbb2370143744ab881a54d39c33ababa 2024-12-03T09:16:37,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/fbb2370143744ab881a54d39c33ababa, entries=150, sequenceid=229, filesize=11.9 K 2024-12-03T09:16:37,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 750235cb928bf724493a7da0f57c7eb3 in 1652ms, sequenceid=229, compaction requested=true 2024-12-03T09:16:37,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:37,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:37,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:37,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:37,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:37,524 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:37,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:37,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:37,524 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:37,525 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:37,525 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133474 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:37,525 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/B is initiating minor compaction (all files) 2024-12-03T09:16:37,525 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/B in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:37,525 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/30ad4c17b8c04a4c837fc57ef75c1f3b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/353598b1c2fa40818a93db6066a0c4e7, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/2c702a326d40401e983820a1cd2355b4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/65747ca4fba844bc85b090a20b3537ce] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=45.5 K 2024-12-03T09:16:37,525 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/A is initiating minor compaction (all files) 2024-12-03T09:16:37,525 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/A in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:37,525 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f81972fea845425da7b944af46229e11, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/05d502f3616049ea8a2b8a1a7f23dafc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/447eb4b9845c44ec934d871db504ad07, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e4a016aa41b5479f8790e56dfa48ad97] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=130.3 K 2024-12-03T09:16:37,526 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:37,526 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 30ad4c17b8c04a4c837fc57ef75c1f3b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733217390439 2024-12-03T09:16:37,526 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f81972fea845425da7b944af46229e11, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/05d502f3616049ea8a2b8a1a7f23dafc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/447eb4b9845c44ec934d871db504ad07, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e4a016aa41b5479f8790e56dfa48ad97] 2024-12-03T09:16:37,526 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 353598b1c2fa40818a93db6066a0c4e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733217391598 2024-12-03T09:16:37,526 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c702a326d40401e983820a1cd2355b4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733217391697 2024-12-03T09:16:37,526 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 65747ca4fba844bc85b090a20b3537ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733217393781 2024-12-03T09:16:37,526 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting f81972fea845425da7b944af46229e11, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733217390439 2024-12-03T09:16:37,527 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05d502f3616049ea8a2b8a1a7f23dafc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733217391598 2024-12-03T09:16:37,527 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 447eb4b9845c44ec934d871db504ad07, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733217391697 2024-12-03T09:16:37,527 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4a016aa41b5479f8790e56dfa48ad97, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733217393781 2024-12-03T09:16:37,533 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:37,533 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#B#compaction#479 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:37,533 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/50054ea6dd0041e49483ab9ebc420fda is 50, key is test_row_0/B:col10/1733217393781/Put/seqid=0 2024-12-03T09:16:37,534 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120372a3b046a32f4bd295d6c2d2dd70369f_750235cb928bf724493a7da0f57c7eb3 store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:37,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742396_1572 (size=12697) 2024-12-03T09:16:37,537 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120372a3b046a32f4bd295d6c2d2dd70369f_750235cb928bf724493a7da0f57c7eb3, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:37,537 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120372a3b046a32f4bd295d6c2d2dd70369f_750235cb928bf724493a7da0f57c7eb3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:37,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742397_1573 (size=4469) 2024-12-03T09:16:37,540 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#A#compaction#480 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:37,541 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/b24547fcbb464d40b962a6e9d4fdc64b is 175, key is test_row_0/A:col10/1733217393781/Put/seqid=0 2024-12-03T09:16:37,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742398_1574 (size=31651) 2024-12-03T09:16:37,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-03T09:16:37,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:37,609 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T09:16:37,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:37,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:37,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:37,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:37,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:37,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:37,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e6f1423ab4634caa82668831377ff01f_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217395894/Put/seqid=0 2024-12-03T09:16:37,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742399_1575 (size=12304) 2024-12-03T09:16:37,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:37,634 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e6f1423ab4634caa82668831377ff01f_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e6f1423ab4634caa82668831377ff01f_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:37,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/b8801de33a834db1a50856e9467f3d52, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:37,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/b8801de33a834db1a50856e9467f3d52 is 175, key is test_row_0/A:col10/1733217395894/Put/seqid=0 2024-12-03T09:16:37,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742400_1576 (size=31105) 2024-12-03T09:16:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T09:16:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:38,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:38,013 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/50054ea6dd0041e49483ab9ebc420fda as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/50054ea6dd0041e49483ab9ebc420fda 2024-12-03T09:16:38,025 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/b24547fcbb464d40b962a6e9d4fdc64b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/b24547fcbb464d40b962a6e9d4fdc64b 2024-12-03T09:16:38,028 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/B of 750235cb928bf724493a7da0f57c7eb3 into 50054ea6dd0041e49483ab9ebc420fda(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:38,028 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:38,028 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/B, priority=12, startTime=1733217397524; duration=0sec 2024-12-03T09:16:38,028 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:38,028 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:B 2024-12-03T09:16:38,028 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:38,029 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/A of 750235cb928bf724493a7da0f57c7eb3 into b24547fcbb464d40b962a6e9d4fdc64b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:38,029 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:38,029 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/A, priority=12, startTime=1733217397524; duration=0sec 2024-12-03T09:16:38,029 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:38,029 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:A 2024-12-03T09:16:38,029 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:38,029 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/C is initiating minor compaction (all files) 2024-12-03T09:16:38,029 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/C in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:38,030 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/e1215d9d93a44663b861d61b6cc8a344, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/3a1790c3ec364cec9ff6968831ace777, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/556be96e1fa04f28b80fbf41d7adfd5a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/fbb2370143744ab881a54d39c33ababa] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=45.5 K 2024-12-03T09:16:38,031 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting e1215d9d93a44663b861d61b6cc8a344, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733217390439 2024-12-03T09:16:38,031 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a1790c3ec364cec9ff6968831ace777, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733217391598 2024-12-03T09:16:38,031 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 556be96e1fa04f28b80fbf41d7adfd5a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733217391697 2024-12-03T09:16:38,031 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb2370143744ab881a54d39c33ababa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733217393781 2024-12-03T09:16:38,037 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#C#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:38,037 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/6b778751b0c5439fabf825fb864ac84a is 50, key is test_row_0/C:col10/1733217393781/Put/seqid=0 2024-12-03T09:16:38,039 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/b8801de33a834db1a50856e9467f3d52 2024-12-03T09:16:38,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217458034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217458037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217458039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/41cacd44c8df4e0c9f2fd0e93d672f59 is 50, key is test_row_0/B:col10/1733217395894/Put/seqid=0 2024-12-03T09:16:38,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742401_1577 (size=12697) 2024-12-03T09:16:38,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217458050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,052 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_0' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742402_1578 (size=12151) 2024-12-03T09:16:38,058 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/41cacd44c8df4e0c9f2fd0e93d672f59 2024-12-03T09:16:38,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217458059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,061 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/344312c578ef4d64b21383274720cf7a is 50, key is test_row_0/C:col10/1733217395894/Put/seqid=0 2024-12-03T09:16:38,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742403_1579 (size=12151) 2024-12-03T09:16:38,068 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/344312c578ef4d64b21383274720cf7a 2024-12-03T09:16:38,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/b8801de33a834db1a50856e9467f3d52 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/b8801de33a834db1a50856e9467f3d52 2024-12-03T09:16:38,077 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/b8801de33a834db1a50856e9467f3d52, entries=150, sequenceid=244, filesize=30.4 K 2024-12-03T09:16:38,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/41cacd44c8df4e0c9f2fd0e93d672f59 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/41cacd44c8df4e0c9f2fd0e93d672f59 2024-12-03T09:16:38,081 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/41cacd44c8df4e0c9f2fd0e93d672f59, entries=150, sequenceid=244, filesize=11.9 K 2024-12-03T09:16:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/344312c578ef4d64b21383274720cf7a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/344312c578ef4d64b21383274720cf7a 2024-12-03T09:16:38,086 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/344312c578ef4d64b21383274720cf7a, entries=150, sequenceid=244, filesize=11.9 K 2024-12-03T09:16:38,087 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 750235cb928bf724493a7da0f57c7eb3 in 478ms, sequenceid=244, compaction requested=false 2024-12-03T09:16:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:38,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-03T09:16:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-03T09:16:38,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-03T09:16:38,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3150 sec 2024-12-03T09:16:38,091 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.3200 sec 2024-12-03T09:16:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:38,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T09:16:38,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:38,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:38,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:38,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:38,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:38,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:38,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e87f6568c52d491691f0ed4c0ee0d04f_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217398151/Put/seqid=0 2024-12-03T09:16:38,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742404_1580 (size=14994) 2024-12-03T09:16:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217458167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217458167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217458168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217458279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217458279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217458279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,452 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/6b778751b0c5439fabf825fb864ac84a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6b778751b0c5439fabf825fb864ac84a 2024-12-03T09:16:38,456 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/C of 750235cb928bf724493a7da0f57c7eb3 into 6b778751b0c5439fabf825fb864ac84a(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:38,456 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:38,456 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/C, priority=12, startTime=1733217397524; duration=0sec 2024-12-03T09:16:38,456 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:38,456 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:C 2024-12-03T09:16:38,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217458489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217458489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217458489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,564 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:38,567 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e87f6568c52d491691f0ed4c0ee0d04f_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e87f6568c52d491691f0ed4c0ee0d04f_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:38,568 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/4ac0aae24e6445c6a32599016cf31bfa, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:38,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/4ac0aae24e6445c6a32599016cf31bfa is 175, key is test_row_0/A:col10/1733217398151/Put/seqid=0 2024-12-03T09:16:38,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742405_1581 (size=39949) 2024-12-03T09:16:38,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217458799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217458799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217458800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:38,972 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/4ac0aae24e6445c6a32599016cf31bfa 2024-12-03T09:16:38,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/5353751367d94a56964b262bc3fa1b4c is 50, key is test_row_0/B:col10/1733217398151/Put/seqid=0 2024-12-03T09:16:38,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742406_1582 (size=12301) 2024-12-03T09:16:39,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:39,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217459309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217459309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:39,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217459310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:39,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/5353751367d94a56964b262bc3fa1b4c 2024-12-03T09:16:39,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/5cdd74f59a4b4128bf89e31d79d42f38 is 50, key is test_row_0/C:col10/1733217398151/Put/seqid=0 2024-12-03T09:16:39,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742407_1583 (size=12301) 2024-12-03T09:16:39,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/5cdd74f59a4b4128bf89e31d79d42f38 2024-12-03T09:16:39,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/4ac0aae24e6445c6a32599016cf31bfa as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4ac0aae24e6445c6a32599016cf31bfa 2024-12-03T09:16:39,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4ac0aae24e6445c6a32599016cf31bfa, entries=200, sequenceid=265, filesize=39.0 K 2024-12-03T09:16:39,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/5353751367d94a56964b262bc3fa1b4c as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/5353751367d94a56964b262bc3fa1b4c 2024-12-03T09:16:39,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/5353751367d94a56964b262bc3fa1b4c, entries=150, sequenceid=265, filesize=12.0 K 2024-12-03T09:16:39,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/5cdd74f59a4b4128bf89e31d79d42f38 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/5cdd74f59a4b4128bf89e31d79d42f38 2024-12-03T09:16:39,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/5cdd74f59a4b4128bf89e31d79d42f38, entries=150, sequenceid=265, filesize=12.0 K 2024-12-03T09:16:39,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 750235cb928bf724493a7da0f57c7eb3 in 1654ms, sequenceid=265, compaction requested=true 2024-12-03T09:16:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:39,806 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:39,806 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:39,807 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:39,807 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:39,807 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/B is initiating minor compaction (all files) 2024-12-03T09:16:39,807 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/A is initiating minor compaction (all files) 2024-12-03T09:16:39,807 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/B in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:39,807 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/A in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:39,807 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/50054ea6dd0041e49483ab9ebc420fda, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/41cacd44c8df4e0c9f2fd0e93d672f59, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/5353751367d94a56964b262bc3fa1b4c] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=36.3 K 2024-12-03T09:16:39,807 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/b24547fcbb464d40b962a6e9d4fdc64b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/b8801de33a834db1a50856e9467f3d52, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4ac0aae24e6445c6a32599016cf31bfa] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=100.3 K 2024-12-03T09:16:39,807 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:39,807 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/b24547fcbb464d40b962a6e9d4fdc64b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/b8801de33a834db1a50856e9467f3d52, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4ac0aae24e6445c6a32599016cf31bfa] 2024-12-03T09:16:39,807 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 50054ea6dd0041e49483ab9ebc420fda, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733217393781 2024-12-03T09:16:39,807 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b24547fcbb464d40b962a6e9d4fdc64b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733217393781 2024-12-03T09:16:39,807 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 41cacd44c8df4e0c9f2fd0e93d672f59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733217395892 2024-12-03T09:16:39,808 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8801de33a834db1a50856e9467f3d52, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733217395892 2024-12-03T09:16:39,808 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5353751367d94a56964b262bc3fa1b4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733217398033 2024-12-03T09:16:39,808 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ac0aae24e6445c6a32599016cf31bfa, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733217398032 2024-12-03T09:16:39,813 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:39,814 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#B#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:39,814 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/38d0988172634320a7ca9c4ca3c6608a is 50, key is test_row_0/B:col10/1733217398151/Put/seqid=0 2024-12-03T09:16:39,817 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203fc1560372f9d4ce58af9b7f80030d820_750235cb928bf724493a7da0f57c7eb3 store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:39,818 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203fc1560372f9d4ce58af9b7f80030d820_750235cb928bf724493a7da0f57c7eb3, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:39,819 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fc1560372f9d4ce58af9b7f80030d820_750235cb928bf724493a7da0f57c7eb3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:39,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742408_1584 (size=12949) 2024-12-03T09:16:39,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742409_1585 (size=4469) 2024-12-03T09:16:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T09:16:39,930 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:39,931 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-03T09:16:39,933 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T09:16:39,934 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:39,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T09:16:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:40,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T09:16:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:40,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:40,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120316c2470fc5344709b3d51196d4c40690_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217400061/Put/seqid=0 2024-12-03T09:16:40,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742410_1586 (size=14994) 2024-12-03T09:16:40,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T09:16:40,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:40,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:40,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:40,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217460108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:40,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217460109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:40,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217460219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:40,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:40,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217460219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:40,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:40,225 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/38d0988172634320a7ca9c4ca3c6608a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/38d0988172634320a7ca9c4ca3c6608a 2024-12-03T09:16:40,229 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/B of 750235cb928bf724493a7da0f57c7eb3 into 38d0988172634320a7ca9c4ca3c6608a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:40,229 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:40,229 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/B, priority=13, startTime=1733217399806; duration=0sec 2024-12-03T09:16:40,229 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:40,229 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:B 2024-12-03T09:16:40,230 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T09:16:40,230 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#A#compaction#489 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:40,230 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/70c5c1e1d83b41739db4965f6204532a is 175, key is test_row_0/A:col10/1733217398151/Put/seqid=0 2024-12-03T09:16:40,232 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T09:16:40,232 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/C is initiating minor compaction (all files) 2024-12-03T09:16:40,232 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/C in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:40,232 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6b778751b0c5439fabf825fb864ac84a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/344312c578ef4d64b21383274720cf7a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/5cdd74f59a4b4128bf89e31d79d42f38] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=36.3 K 2024-12-03T09:16:40,232 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b778751b0c5439fabf825fb864ac84a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733217393781 2024-12-03T09:16:40,233 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 344312c578ef4d64b21383274720cf7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733217395892 2024-12-03T09:16:40,233 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cdd74f59a4b4128bf89e31d79d42f38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733217398033 2024-12-03T09:16:40,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742411_1587 (size=31903) 2024-12-03T09:16:40,237 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#C#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:40,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T09:16:40,238 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/a8858734052749ce95f77c5d90b6e6ab is 50, key is test_row_0/C:col10/1733217398151/Put/seqid=0 2024-12-03T09:16:40,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:40,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:40,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:40,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742412_1588 (size=12949) 2024-12-03T09:16:40,245 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/a8858734052749ce95f77c5d90b6e6ab as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/a8858734052749ce95f77c5d90b6e6ab 2024-12-03T09:16:40,248 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/C of 750235cb928bf724493a7da0f57c7eb3 into a8858734052749ce95f77c5d90b6e6ab(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:40,248 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:40,248 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/C, priority=13, startTime=1733217399806; duration=0sec 2024-12-03T09:16:40,249 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:40,249 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:C 2024-12-03T09:16:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T09:16:40,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217460319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:40,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217460319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:40,321 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:40,322 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:40,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217460329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:40,331 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:40,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T09:16:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:40,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:40,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217460429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217460429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:40,479 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:40,482 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120316c2470fc5344709b3d51196d4c40690_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120316c2470fc5344709b3d51196d4c40690_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:40,482 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/3cc5bc14de8c485dabf3fde4533feb27, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:40,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/3cc5bc14de8c485dabf3fde4533feb27 is 175, key is test_row_0/A:col10/1733217400061/Put/seqid=0 2024-12-03T09:16:40,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742413_1589 (size=39949) 2024-12-03T09:16:40,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T09:16:40,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:40,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:40,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:40,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T09:16:40,637 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/70c5c1e1d83b41739db4965f6204532a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/70c5c1e1d83b41739db4965f6204532a 2024-12-03T09:16:40,640 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/A of 750235cb928bf724493a7da0f57c7eb3 into 70c5c1e1d83b41739db4965f6204532a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:40,640 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:40,640 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/A, priority=13, startTime=1733217399806; duration=0sec 2024-12-03T09:16:40,640 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:40,640 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:A 2024-12-03T09:16:40,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T09:16:40,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:40,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:40,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:40,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:40,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217460740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217460740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:40,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:40,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T09:16:40,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:40,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:40,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:40,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:40,886 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/3cc5bc14de8c485dabf3fde4533feb27 2024-12-03T09:16:40,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/3e7d0dc8754e4196be850727bdc3a881 is 50, key is test_row_0/B:col10/1733217400061/Put/seqid=0 2024-12-03T09:16:40,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742414_1590 (size=12301) 2024-12-03T09:16:41,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T09:16:41,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:41,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:41,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:41,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:41,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T09:16:41,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T09:16:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:41,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:41,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:41,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217461249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:41,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217461249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:41,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/3e7d0dc8754e4196be850727bdc3a881 2024-12-03T09:16:41,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/101f2cee823a47059c0b8fe59a8e9f11 is 50, key is test_row_0/C:col10/1733217400061/Put/seqid=0 2024-12-03T09:16:41,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742415_1591 (size=12301) 2024-12-03T09:16:41,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/101f2cee823a47059c0b8fe59a8e9f11 2024-12-03T09:16:41,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/3cc5bc14de8c485dabf3fde4533feb27 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/3cc5bc14de8c485dabf3fde4533feb27 2024-12-03T09:16:41,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/3cc5bc14de8c485dabf3fde4533feb27, entries=200, sequenceid=281, filesize=39.0 K 2024-12-03T09:16:41,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/3e7d0dc8754e4196be850727bdc3a881 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/3e7d0dc8754e4196be850727bdc3a881 2024-12-03T09:16:41,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T09:16:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:41,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:41,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:16:41,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/3e7d0dc8754e4196be850727bdc3a881, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T09:16:41,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/101f2cee823a47059c0b8fe59a8e9f11 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/101f2cee823a47059c0b8fe59a8e9f11 2024-12-03T09:16:41,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/101f2cee823a47059c0b8fe59a8e9f11, entries=150, sequenceid=281, filesize=12.0 K 2024-12-03T09:16:41,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 750235cb928bf724493a7da0f57c7eb3 in 1256ms, sequenceid=281, compaction requested=false 2024-12-03T09:16:41,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:41,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-03T09:16:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:41,462 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T09:16:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:41,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:41,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032cc9e7b7d5034a2f95c558fd2b10a578_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217400106/Put/seqid=0 2024-12-03T09:16:41,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742416_1592 (size=12454) 2024-12-03T09:16:41,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:41,873 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032cc9e7b7d5034a2f95c558fd2b10a578_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032cc9e7b7d5034a2f95c558fd2b10a578_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:41,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/26f64745cbc3491cad67b4319f961cc0, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:41,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/26f64745cbc3491cad67b4319f961cc0 is 175, key is test_row_0/A:col10/1733217400106/Put/seqid=0 2024-12-03T09:16:41,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742417_1593 (size=31255) 2024-12-03T09:16:41,877 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/26f64745cbc3491cad67b4319f961cc0 2024-12-03T09:16:41,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/e2e2874ae35a4c348b81a2a1340df32b is 50, key is test_row_0/B:col10/1733217400106/Put/seqid=0 2024-12-03T09:16:41,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742418_1594 (size=12301) 2024-12-03T09:16:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T09:16:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:42,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:42,285 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/e2e2874ae35a4c348b81a2a1340df32b 2024-12-03T09:16:42,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217462284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:42,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217462287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:42,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/9677350dd5104903a428bbf8cb453fa2 is 50, key is test_row_0/C:col10/1733217400106/Put/seqid=0 2024-12-03T09:16:42,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742419_1595 (size=12301) 2024-12-03T09:16:42,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45470 deadline: 1733217462329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:42,332 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:42,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:42,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45420 deadline: 1733217462339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:42,342 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_2' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:42,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:42,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45424 deadline: 1733217462349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:42,353 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 699c9baf3d29,42271,1733217224446 for 'test_row_1' in 750235cb928bf724493a7da0f57c7eb3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T09:16:42,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:42,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217462389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:42,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217462399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:42,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:42,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217462600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:42,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217462609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:42,696 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/9677350dd5104903a428bbf8cb453fa2 2024-12-03T09:16:42,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/26f64745cbc3491cad67b4319f961cc0 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/26f64745cbc3491cad67b4319f961cc0 2024-12-03T09:16:42,702 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/26f64745cbc3491cad67b4319f961cc0, entries=150, sequenceid=301, filesize=30.5 K 2024-12-03T09:16:42,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/e2e2874ae35a4c348b81a2a1340df32b as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/e2e2874ae35a4c348b81a2a1340df32b 2024-12-03T09:16:42,705 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/e2e2874ae35a4c348b81a2a1340df32b, entries=150, sequenceid=301, filesize=12.0 K 2024-12-03T09:16:42,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/9677350dd5104903a428bbf8cb453fa2 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/9677350dd5104903a428bbf8cb453fa2 2024-12-03T09:16:42,709 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/9677350dd5104903a428bbf8cb453fa2, entries=150, sequenceid=301, filesize=12.0 K 2024-12-03T09:16:42,709 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 750235cb928bf724493a7da0f57c7eb3 in 1247ms, sequenceid=301, compaction requested=true 2024-12-03T09:16:42,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:42,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:42,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-03T09:16:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-03T09:16:42,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-03T09:16:42,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7770 sec 2024-12-03T09:16:42,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.7820 sec 2024-12-03T09:16:42,800 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T09:16:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:42,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T09:16:42,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:42,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:42,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:42,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:42,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:42,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:42,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030f3a98f2f82b4d0197229d479de718a0_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217402910/Put/seqid=0 2024-12-03T09:16:42,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742420_1596 (size=12454) 2024-12-03T09:16:42,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217462939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:42,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217462940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:42,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:42,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:43,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217463049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:43,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217463049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:43,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:43,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:43,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:43,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:43,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:43,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:43,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217463259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:43,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217463259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:43,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:43,319 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:43,322 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030f3a98f2f82b4d0197229d479de718a0_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030f3a98f2f82b4d0197229d479de718a0_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:43,323 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/51d7a80faea44744b251e6c172716fc8, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:43,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/51d7a80faea44744b251e6c172716fc8 is 175, key is test_row_0/A:col10/1733217402910/Put/seqid=0 2024-12-03T09:16:43,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742421_1597 (size=31255) 2024-12-03T09:16:43,327 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/51d7a80faea44744b251e6c172716fc8 2024-12-03T09:16:43,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/98f3409cb6fb400d9fe5834505078e51 is 50, key is test_row_0/B:col10/1733217402910/Put/seqid=0 2024-12-03T09:16:43,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742422_1598 (size=12301) 2024-12-03T09:16:43,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217463569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:43,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217463570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:43,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:43,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:43,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/98f3409cb6fb400d9fe5834505078e51 2024-12-03T09:16:43,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/28eb360761644df0aee4ca5dac225e56 is 50, key is test_row_0/C:col10/1733217402910/Put/seqid=0 2024-12-03T09:16:43,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742423_1599 (size=12301) 2024-12-03T09:16:44,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45456 deadline: 1733217464079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:44,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T09:16:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42271 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45428 deadline: 1733217464079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 , the old value is region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T09:16:44,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., hostname=699c9baf3d29,42271,1733217224446, seqNum=5 because the exception is null or not the one we care about 2024-12-03T09:16:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T09:16:44,100 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:44,100 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-03T09:16:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-03T09:16:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T09:16:44,102 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T09:16:44,103 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T09:16:44,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T09:16:44,149 INFO [Thread-2297 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2297. 2024-12-03T09:16:44,149 DEBUG [Thread-2297 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:44,149 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:44,149 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:44,149 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:44,150 INFO [Thread-2295 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2295. 2024-12-03T09:16:44,150 DEBUG [Thread-2295 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:44,150 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:44,151 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:44,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:44,151 INFO [Thread-2291 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2291. 2024-12-03T09:16:44,151 DEBUG [Thread-2291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:44,151 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:44,151 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:44,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:44,152 INFO [Thread-2299 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2299. 2024-12-03T09:16:44,153 DEBUG [Thread-2299 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:44,153 DEBUG [Thread-2299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:44,153 DEBUG [Thread-2299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:44,153 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:44,154 INFO [Thread-2293 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2293. 2024-12-03T09:16:44,154 DEBUG [Thread-2293 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:44,154 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:44,154 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:44,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:44,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/28eb360761644df0aee4ca5dac225e56 2024-12-03T09:16:44,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/51d7a80faea44744b251e6c172716fc8 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/51d7a80faea44744b251e6c172716fc8 2024-12-03T09:16:44,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/51d7a80faea44744b251e6c172716fc8, entries=150, sequenceid=317, filesize=30.5 K 2024-12-03T09:16:44,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/98f3409cb6fb400d9fe5834505078e51 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/98f3409cb6fb400d9fe5834505078e51 2024-12-03T09:16:44,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/98f3409cb6fb400d9fe5834505078e51, entries=150, sequenceid=317, filesize=12.0 K 2024-12-03T09:16:44,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/28eb360761644df0aee4ca5dac225e56 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/28eb360761644df0aee4ca5dac225e56 2024-12-03T09:16:44,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/28eb360761644df0aee4ca5dac225e56, entries=150, sequenceid=317, filesize=12.0 K 2024-12-03T09:16:44,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 750235cb928bf724493a7da0f57c7eb3 in 1259ms, sequenceid=317, compaction requested=true 2024-12-03T09:16:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T09:16:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T09:16:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:44,170 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 750235cb928bf724493a7da0f57c7eb3:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T09:16:44,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:44,170 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:44,171 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:44,171 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:44,171 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/B is initiating minor compaction (all files) 2024-12-03T09:16:44,171 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/A is initiating minor compaction (all files) 2024-12-03T09:16:44,171 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/B in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:44,171 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/A in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:44,171 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/38d0988172634320a7ca9c4ca3c6608a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/3e7d0dc8754e4196be850727bdc3a881, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/e2e2874ae35a4c348b81a2a1340df32b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/98f3409cb6fb400d9fe5834505078e51] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=48.7 K 2024-12-03T09:16:44,171 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/70c5c1e1d83b41739db4965f6204532a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/3cc5bc14de8c485dabf3fde4533feb27, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/26f64745cbc3491cad67b4319f961cc0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/51d7a80faea44744b251e6c172716fc8] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=131.2 K 2024-12-03T09:16:44,171 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:44,171 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. files: [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/70c5c1e1d83b41739db4965f6204532a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/3cc5bc14de8c485dabf3fde4533feb27, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/26f64745cbc3491cad67b4319f961cc0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/51d7a80faea44744b251e6c172716fc8] 2024-12-03T09:16:44,171 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 38d0988172634320a7ca9c4ca3c6608a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733217398033 2024-12-03T09:16:44,171 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e7d0dc8754e4196be850727bdc3a881, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733217398166 2024-12-03T09:16:44,171 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70c5c1e1d83b41739db4965f6204532a, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733217398033 2024-12-03T09:16:44,172 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cc5bc14de8c485dabf3fde4533feb27, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733217398166 2024-12-03T09:16:44,172 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting e2e2874ae35a4c348b81a2a1340df32b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733217400100 2024-12-03T09:16:44,172 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26f64745cbc3491cad67b4319f961cc0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733217400100 2024-12-03T09:16:44,172 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f3409cb6fb400d9fe5834505078e51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217402281 2024-12-03T09:16:44,172 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51d7a80faea44744b251e6c172716fc8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217402281 2024-12-03T09:16:44,176 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:44,177 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#B#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:44,177 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203bf3a2348b75f4faebdf8dfd0c1143846_750235cb928bf724493a7da0f57c7eb3 store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:44,177 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/09e9945ae5424a82afee2430a23da597 is 50, key is test_row_0/B:col10/1733217402910/Put/seqid=0 2024-12-03T09:16:44,181 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203bf3a2348b75f4faebdf8dfd0c1143846_750235cb928bf724493a7da0f57c7eb3, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:44,181 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bf3a2348b75f4faebdf8dfd0c1143846_750235cb928bf724493a7da0f57c7eb3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:44,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742424_1600 (size=13085) 2024-12-03T09:16:44,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742425_1601 (size=4469) 2024-12-03T09:16:44,190 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#A#compaction#501 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:44,190 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/dd58ee07bfc44ac289eda68b43451378 is 175, key is test_row_0/A:col10/1733217402910/Put/seqid=0 2024-12-03T09:16:44,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742426_1602 (size=32039) 2024-12-03T09:16:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T09:16:44,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42271 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-03T09:16:44,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:44,255 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T09:16:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:44,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038534a2bcb0ac40cda696f288f0d393f3_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217402935/Put/seqid=0 2024-12-03T09:16:44,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742427_1603 (size=12454) 2024-12-03T09:16:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T09:16:44,591 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/09e9945ae5424a82afee2430a23da597 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/09e9945ae5424a82afee2430a23da597 2024-12-03T09:16:44,596 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/B of 750235cb928bf724493a7da0f57c7eb3 into 09e9945ae5424a82afee2430a23da597(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:44,596 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:44,596 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/B, priority=12, startTime=1733217404170; duration=0sec 2024-12-03T09:16:44,596 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T09:16:44,596 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:B 2024-12-03T09:16:44,596 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:16:44,597 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T09:16:44,597 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1541): 750235cb928bf724493a7da0f57c7eb3/C is initiating minor compaction (all files) 2024-12-03T09:16:44,597 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 750235cb928bf724493a7da0f57c7eb3/C in TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:44,597 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/dd58ee07bfc44ac289eda68b43451378 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/dd58ee07bfc44ac289eda68b43451378 2024-12-03T09:16:44,597 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/a8858734052749ce95f77c5d90b6e6ab, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/101f2cee823a47059c0b8fe59a8e9f11, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/9677350dd5104903a428bbf8cb453fa2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/28eb360761644df0aee4ca5dac225e56] into tmpdir=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp, totalSize=48.7 K 2024-12-03T09:16:44,598 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting a8858734052749ce95f77c5d90b6e6ab, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733217398033 2024-12-03T09:16:44,598 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 101f2cee823a47059c0b8fe59a8e9f11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733217398166 2024-12-03T09:16:44,598 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 9677350dd5104903a428bbf8cb453fa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733217400100 2024-12-03T09:16:44,598 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] compactions.Compactor(225): Compacting 28eb360761644df0aee4ca5dac225e56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1733217402281 2024-12-03T09:16:44,601 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/A of 750235cb928bf724493a7da0f57c7eb3 into dd58ee07bfc44ac289eda68b43451378(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:44,601 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:44,601 INFO [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/A, priority=12, startTime=1733217404170; duration=0sec 2024-12-03T09:16:44,601 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:44,601 DEBUG [RS:0;699c9baf3d29:42271-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:A 2024-12-03T09:16:44,606 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 750235cb928bf724493a7da0f57c7eb3#C#compaction#503 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:16:44,607 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/ba8458a098a34825addb753bca41f39a is 50, key is test_row_0/C:col10/1733217402910/Put/seqid=0 2024-12-03T09:16:44,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742428_1604 (size=13085) 2024-12-03T09:16:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:44,670 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038534a2bcb0ac40cda696f288f0d393f3_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038534a2bcb0ac40cda696f288f0d393f3_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:44,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/f8092073fa06493996334cedba91dd35, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:44,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/f8092073fa06493996334cedba91dd35 is 175, key is test_row_0/A:col10/1733217402935/Put/seqid=0 2024-12-03T09:16:44,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742429_1605 (size=31255) 2024-12-03T09:16:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T09:16:45,021 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/ba8458a098a34825addb753bca41f39a as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/ba8458a098a34825addb753bca41f39a 2024-12-03T09:16:45,025 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 750235cb928bf724493a7da0f57c7eb3/C of 750235cb928bf724493a7da0f57c7eb3 into ba8458a098a34825addb753bca41f39a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:16:45,025 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:45,025 INFO [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., storeName=750235cb928bf724493a7da0f57c7eb3/C, priority=12, startTime=1733217404170; duration=0sec 2024-12-03T09:16:45,025 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T09:16:45,025 DEBUG [RS:0;699c9baf3d29:42271-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 750235cb928bf724493a7da0f57c7eb3:C 2024-12-03T09:16:45,075 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/f8092073fa06493996334cedba91dd35 2024-12-03T09:16:45,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/4234920d42274081a56798d1c06ac8c1 is 50, key is test_row_0/B:col10/1733217402935/Put/seqid=0 2024-12-03T09:16:45,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742430_1606 (size=12301) 2024-12-03T09:16:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] regionserver.HRegion(8855): Flush requested on 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:45,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. as already flushing 2024-12-03T09:16:45,101 INFO [Thread-2286 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2286. 2024-12-03T09:16:45,101 DEBUG [Thread-2286 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:45,101 INFO [Thread-2284 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2284. 2024-12-03T09:16:45,101 DEBUG [Thread-2286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:45,101 DEBUG [Thread-2284 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:45,101 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:45,101 DEBUG [Thread-2286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:45,101 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:45,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:45,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T09:16:45,489 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/4234920d42274081a56798d1c06ac8c1 2024-12-03T09:16:45,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/7ce8bed9b3124b14a58bd1cbbef8f7dc is 50, key is test_row_0/C:col10/1733217402935/Put/seqid=0 2024-12-03T09:16:45,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742431_1607 (size=12301) 2024-12-03T09:16:45,906 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/7ce8bed9b3124b14a58bd1cbbef8f7dc 2024-12-03T09:16:45,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/f8092073fa06493996334cedba91dd35 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f8092073fa06493996334cedba91dd35 2024-12-03T09:16:45,922 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f8092073fa06493996334cedba91dd35, entries=150, sequenceid=333, filesize=30.5 K 2024-12-03T09:16:45,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/4234920d42274081a56798d1c06ac8c1 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/4234920d42274081a56798d1c06ac8c1 2024-12-03T09:16:45,926 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/4234920d42274081a56798d1c06ac8c1, entries=150, sequenceid=333, filesize=12.0 K 2024-12-03T09:16:45,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/7ce8bed9b3124b14a58bd1cbbef8f7dc as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/7ce8bed9b3124b14a58bd1cbbef8f7dc 2024-12-03T09:16:45,930 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/7ce8bed9b3124b14a58bd1cbbef8f7dc, entries=150, sequenceid=333, filesize=12.0 K 2024-12-03T09:16:45,931 INFO [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=13.42 KB/13740 for 750235cb928bf724493a7da0f57c7eb3 in 1675ms, sequenceid=333, compaction requested=false 2024-12-03T09:16:45,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 750235cb928bf724493a7da0f57c7eb3: 2024-12-03T09:16:45,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:45,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-03T09:16:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-03T09:16:45,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-03T09:16:45,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8290 sec 2024-12-03T09:16:45,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8340 sec 2024-12-03T09:16:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T09:16:46,250 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:46,344 INFO [Thread-2280 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2280. 2024-12-03T09:16:46,344 DEBUG [Thread-2280 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:46,344 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:46,344 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:46,344 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:46,350 INFO [Thread-2282 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2282. 2024-12-03T09:16:46,350 DEBUG [Thread-2282 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:46,350 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:46,350 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:46,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:46,401 INFO [Thread-2288 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2288. 2024-12-03T09:16:46,401 DEBUG [Thread-2288 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T09:16:46,401 DEBUG [Thread-2288 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:46,401 DEBUG [Thread-2288 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:46,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:46,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T09:16:46,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-03T09:16:46,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-12-03T09:16:46,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-12-03T09:16:46,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-03T09:16:46,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-03T09:16:46,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T09:16:46,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4326 2024-12-03T09:16:46,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4449 2024-12-03T09:16:46,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4367 2024-12-03T09:16:46,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4340 2024-12-03T09:16:46,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4437 2024-12-03T09:16:46,402 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T09:16:46,402 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T09:16:46,402 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:16:46,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:46,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:46,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:46,403 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-03T09:16:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T09:16:46,408 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217406408"}]},"ts":"1733217406408"} 2024-12-03T09:16:46,410 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T09:16:46,410 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T09:16:46,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T09:16:46,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, UNASSIGN}] 2024-12-03T09:16:46,414 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, UNASSIGN 2024-12-03T09:16:46,415 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=750235cb928bf724493a7da0f57c7eb3, regionState=CLOSING, regionLocation=699c9baf3d29,42271,1733217224446 2024-12-03T09:16:46,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, UNASSIGN because future has completed 2024-12-03T09:16:46,416 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T09:16:46,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446}] 2024-12-03T09:16:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T09:16:46,569 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:46,569 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T09:16:46,569 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 750235cb928bf724493a7da0f57c7eb3, disabling compactions & flushes 2024-12-03T09:16:46,569 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:46,569 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:46,569 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. after waiting 0 ms 2024-12-03T09:16:46,569 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:46,570 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 750235cb928bf724493a7da0f57c7eb3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T09:16:46,570 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=A 2024-12-03T09:16:46,570 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:46,570 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=B 2024-12-03T09:16:46,570 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:46,570 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 750235cb928bf724493a7da0f57c7eb3, store=C 2024-12-03T09:16:46,570 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T09:16:46,580 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203dbed9b524f8f4a8d837704bba97e5fd5_750235cb928bf724493a7da0f57c7eb3 is 50, key is test_row_0/A:col10/1733217405100/Put/seqid=0 2024-12-03T09:16:46,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742432_1608 (size=12454) 2024-12-03T09:16:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T09:16:46,984 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:16:46,990 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203dbed9b524f8f4a8d837704bba97e5fd5_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dbed9b524f8f4a8d837704bba97e5fd5_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:46,991 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/1ba6b30343534713aff700b39b8b354f, store: [table=TestAcidGuarantees family=A region=750235cb928bf724493a7da0f57c7eb3] 2024-12-03T09:16:46,992 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/1ba6b30343534713aff700b39b8b354f is 175, key is test_row_0/A:col10/1733217405100/Put/seqid=0 2024-12-03T09:16:46,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742433_1609 (size=31255) 2024-12-03T09:16:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T09:16:47,398 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/1ba6b30343534713aff700b39b8b354f 2024-12-03T09:16:47,413 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/699372d083f04b92b2994bca559142fb is 50, key is test_row_0/B:col10/1733217405100/Put/seqid=0 2024-12-03T09:16:47,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742434_1610 (size=12301) 2024-12-03T09:16:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T09:16:47,818 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/699372d083f04b92b2994bca559142fb 2024-12-03T09:16:47,829 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/1029afe72195407da55cf6e5e29c00db is 50, key is test_row_0/C:col10/1733217405100/Put/seqid=0 2024-12-03T09:16:47,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742435_1611 (size=12301) 2024-12-03T09:16:48,234 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/1029afe72195407da55cf6e5e29c00db 2024-12-03T09:16:48,243 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/A/1ba6b30343534713aff700b39b8b354f as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/1ba6b30343534713aff700b39b8b354f 2024-12-03T09:16:48,250 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/1ba6b30343534713aff700b39b8b354f, entries=150, sequenceid=344, filesize=30.5 K 2024-12-03T09:16:48,251 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/B/699372d083f04b92b2994bca559142fb as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/699372d083f04b92b2994bca559142fb 2024-12-03T09:16:48,254 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/699372d083f04b92b2994bca559142fb, entries=150, sequenceid=344, filesize=12.0 K 2024-12-03T09:16:48,255 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/.tmp/C/1029afe72195407da55cf6e5e29c00db as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/1029afe72195407da55cf6e5e29c00db 2024-12-03T09:16:48,258 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/1029afe72195407da55cf6e5e29c00db, entries=150, sequenceid=344, filesize=12.0 K 2024-12-03T09:16:48,259 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 750235cb928bf724493a7da0f57c7eb3 in 1690ms, sequenceid=344, compaction requested=true 2024-12-03T09:16:48,259 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e82b08caef46427cb6158098b48a83d2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/19d2aa9ea525465ea56c97a094568b7a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/9cda3555ebd64c4a9bff089d7fe701f8, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/8d46ca390a1f467b9b15f435c672e6fb, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/baaa7cb31d514c158f3395edacc0e9dd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/bcd9a2c850ac40de92b878fbb0c0ab3a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/893666939fba46569441f9aca9e55284, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/5c753c98913c46c18f180a545de3273e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4b8dacad9c49434ba7689a481985a765, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f8cba1dc883e4cb58164a94a8dae779d, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/ca8de862f72443beba56183fde0cfdad, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4c0147242a854935baa5445cacbc2c6b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f81972fea845425da7b944af46229e11, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/05d502f3616049ea8a2b8a1a7f23dafc, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/447eb4b9845c44ec934d871db504ad07, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e4a016aa41b5479f8790e56dfa48ad97, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/b24547fcbb464d40b962a6e9d4fdc64b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/b8801de33a834db1a50856e9467f3d52, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4ac0aae24e6445c6a32599016cf31bfa, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/70c5c1e1d83b41739db4965f6204532a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/3cc5bc14de8c485dabf3fde4533feb27, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/26f64745cbc3491cad67b4319f961cc0, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/51d7a80faea44744b251e6c172716fc8] to archive 2024-12-03T09:16:48,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:16:48,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e82b08caef46427cb6158098b48a83d2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e82b08caef46427cb6158098b48a83d2 2024-12-03T09:16:48,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/19d2aa9ea525465ea56c97a094568b7a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/19d2aa9ea525465ea56c97a094568b7a 2024-12-03T09:16:48,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/9cda3555ebd64c4a9bff089d7fe701f8 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/9cda3555ebd64c4a9bff089d7fe701f8 2024-12-03T09:16:48,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/8d46ca390a1f467b9b15f435c672e6fb to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/8d46ca390a1f467b9b15f435c672e6fb 2024-12-03T09:16:48,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/baaa7cb31d514c158f3395edacc0e9dd to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/baaa7cb31d514c158f3395edacc0e9dd 2024-12-03T09:16:48,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/bcd9a2c850ac40de92b878fbb0c0ab3a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/bcd9a2c850ac40de92b878fbb0c0ab3a 2024-12-03T09:16:48,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/893666939fba46569441f9aca9e55284 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/893666939fba46569441f9aca9e55284 2024-12-03T09:16:48,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/5c753c98913c46c18f180a545de3273e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/5c753c98913c46c18f180a545de3273e 2024-12-03T09:16:48,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4b8dacad9c49434ba7689a481985a765 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4b8dacad9c49434ba7689a481985a765 2024-12-03T09:16:48,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f8cba1dc883e4cb58164a94a8dae779d to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f8cba1dc883e4cb58164a94a8dae779d 2024-12-03T09:16:48,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/ca8de862f72443beba56183fde0cfdad to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/ca8de862f72443beba56183fde0cfdad 2024-12-03T09:16:48,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4c0147242a854935baa5445cacbc2c6b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4c0147242a854935baa5445cacbc2c6b 2024-12-03T09:16:48,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f81972fea845425da7b944af46229e11 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f81972fea845425da7b944af46229e11 2024-12-03T09:16:48,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/05d502f3616049ea8a2b8a1a7f23dafc to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/05d502f3616049ea8a2b8a1a7f23dafc 2024-12-03T09:16:48,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/447eb4b9845c44ec934d871db504ad07 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/447eb4b9845c44ec934d871db504ad07 2024-12-03T09:16:48,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e4a016aa41b5479f8790e56dfa48ad97 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/e4a016aa41b5479f8790e56dfa48ad97 2024-12-03T09:16:48,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/b24547fcbb464d40b962a6e9d4fdc64b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/b24547fcbb464d40b962a6e9d4fdc64b 2024-12-03T09:16:48,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/b8801de33a834db1a50856e9467f3d52 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/b8801de33a834db1a50856e9467f3d52 2024-12-03T09:16:48,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4ac0aae24e6445c6a32599016cf31bfa to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/4ac0aae24e6445c6a32599016cf31bfa 2024-12-03T09:16:48,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/70c5c1e1d83b41739db4965f6204532a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/70c5c1e1d83b41739db4965f6204532a 2024-12-03T09:16:48,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/3cc5bc14de8c485dabf3fde4533feb27 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/3cc5bc14de8c485dabf3fde4533feb27 2024-12-03T09:16:48,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/26f64745cbc3491cad67b4319f961cc0 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/26f64745cbc3491cad67b4319f961cc0 2024-12-03T09:16:48,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/51d7a80faea44744b251e6c172716fc8 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/51d7a80faea44744b251e6c172716fc8 2024-12-03T09:16:48,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/1ad4d01f224843fb980ea0288d8960a4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/521efd4a88f04e01a6821463a9991b98, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/2dd82073a2d743b68cd836cfc554ee95, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/63e50e96bdc3478d8c730b87c05c8618, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/55a691f9748240d38abf260dbdab75e6, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/ac0d250a95654826916610a7b911d87a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/6a1bdb8f430148d5912394c6a84dcf34, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/c9d277a98ef34a75be255b53a4165341, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/bdae2a15aea94720a6cdeb40379b84cd, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/7082da66a42942568cb489aee76f8521, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/4b054c1e578e4009ae85cc4dec42432e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/30ad4c17b8c04a4c837fc57ef75c1f3b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/774d1aa18e514bba911564a2a6ab4773, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/353598b1c2fa40818a93db6066a0c4e7, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/2c702a326d40401e983820a1cd2355b4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/50054ea6dd0041e49483ab9ebc420fda, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/65747ca4fba844bc85b090a20b3537ce, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/41cacd44c8df4e0c9f2fd0e93d672f59, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/38d0988172634320a7ca9c4ca3c6608a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/5353751367d94a56964b262bc3fa1b4c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/3e7d0dc8754e4196be850727bdc3a881, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/e2e2874ae35a4c348b81a2a1340df32b, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/98f3409cb6fb400d9fe5834505078e51] to archive 2024-12-03T09:16:48,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:16:48,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/1ad4d01f224843fb980ea0288d8960a4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/1ad4d01f224843fb980ea0288d8960a4 2024-12-03T09:16:48,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/521efd4a88f04e01a6821463a9991b98 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/521efd4a88f04e01a6821463a9991b98 2024-12-03T09:16:48,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/2dd82073a2d743b68cd836cfc554ee95 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/2dd82073a2d743b68cd836cfc554ee95 2024-12-03T09:16:48,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/63e50e96bdc3478d8c730b87c05c8618 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/63e50e96bdc3478d8c730b87c05c8618 2024-12-03T09:16:48,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/55a691f9748240d38abf260dbdab75e6 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/55a691f9748240d38abf260dbdab75e6 2024-12-03T09:16:48,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/ac0d250a95654826916610a7b911d87a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/ac0d250a95654826916610a7b911d87a 2024-12-03T09:16:48,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/6a1bdb8f430148d5912394c6a84dcf34 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/6a1bdb8f430148d5912394c6a84dcf34 2024-12-03T09:16:48,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/c9d277a98ef34a75be255b53a4165341 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/c9d277a98ef34a75be255b53a4165341 2024-12-03T09:16:48,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/bdae2a15aea94720a6cdeb40379b84cd to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/bdae2a15aea94720a6cdeb40379b84cd 2024-12-03T09:16:48,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/7082da66a42942568cb489aee76f8521 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/7082da66a42942568cb489aee76f8521 2024-12-03T09:16:48,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/4b054c1e578e4009ae85cc4dec42432e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/4b054c1e578e4009ae85cc4dec42432e 2024-12-03T09:16:48,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/30ad4c17b8c04a4c837fc57ef75c1f3b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/30ad4c17b8c04a4c837fc57ef75c1f3b 2024-12-03T09:16:48,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/774d1aa18e514bba911564a2a6ab4773 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/774d1aa18e514bba911564a2a6ab4773 2024-12-03T09:16:48,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/353598b1c2fa40818a93db6066a0c4e7 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/353598b1c2fa40818a93db6066a0c4e7 2024-12-03T09:16:48,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/2c702a326d40401e983820a1cd2355b4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/2c702a326d40401e983820a1cd2355b4 2024-12-03T09:16:48,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/50054ea6dd0041e49483ab9ebc420fda to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/50054ea6dd0041e49483ab9ebc420fda 2024-12-03T09:16:48,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/65747ca4fba844bc85b090a20b3537ce to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/65747ca4fba844bc85b090a20b3537ce 2024-12-03T09:16:48,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/41cacd44c8df4e0c9f2fd0e93d672f59 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/41cacd44c8df4e0c9f2fd0e93d672f59 2024-12-03T09:16:48,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/38d0988172634320a7ca9c4ca3c6608a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/38d0988172634320a7ca9c4ca3c6608a 2024-12-03T09:16:48,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/5353751367d94a56964b262bc3fa1b4c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/5353751367d94a56964b262bc3fa1b4c 2024-12-03T09:16:48,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/3e7d0dc8754e4196be850727bdc3a881 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/3e7d0dc8754e4196be850727bdc3a881 2024-12-03T09:16:48,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/e2e2874ae35a4c348b81a2a1340df32b to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/e2e2874ae35a4c348b81a2a1340df32b 2024-12-03T09:16:48,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/98f3409cb6fb400d9fe5834505078e51 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/98f3409cb6fb400d9fe5834505078e51 2024-12-03T09:16:48,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/f3fd8e73e3624aaba310e6f2b93ba958, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/ff08a28d9388422bb7b8178084d39faf, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/2800f5e13f7d438aa88e03a7c12da9fa, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/8f2b1b0f688c43eca63311f45d025341, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/d7211b9021734016a749fe337396983c, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6145b19e879146d889f22498a56573a4, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/76ed6a14fbb547c49d698f3390a05592, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/9aa8f893da484dc39635b0cb0edf3485, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/663d6e1af3df472a9e2261512ed94c87, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/f3da1a78ba37476c936e05f0d549977a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6ba81c8484f74246ab6f62bdf8175347, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/e1215d9d93a44663b861d61b6cc8a344, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/22c8d563137f4a388f6729880d93d18e, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/3a1790c3ec364cec9ff6968831ace777, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/556be96e1fa04f28b80fbf41d7adfd5a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6b778751b0c5439fabf825fb864ac84a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/fbb2370143744ab881a54d39c33ababa, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/344312c578ef4d64b21383274720cf7a, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/a8858734052749ce95f77c5d90b6e6ab, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/5cdd74f59a4b4128bf89e31d79d42f38, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/101f2cee823a47059c0b8fe59a8e9f11, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/9677350dd5104903a428bbf8cb453fa2, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/28eb360761644df0aee4ca5dac225e56] to archive 2024-12-03T09:16:48,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:16:48,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/f3fd8e73e3624aaba310e6f2b93ba958 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/f3fd8e73e3624aaba310e6f2b93ba958 2024-12-03T09:16:48,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/ff08a28d9388422bb7b8178084d39faf to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/ff08a28d9388422bb7b8178084d39faf 2024-12-03T09:16:48,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/2800f5e13f7d438aa88e03a7c12da9fa to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/2800f5e13f7d438aa88e03a7c12da9fa 2024-12-03T09:16:48,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/8f2b1b0f688c43eca63311f45d025341 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/8f2b1b0f688c43eca63311f45d025341 2024-12-03T09:16:48,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/d7211b9021734016a749fe337396983c to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/d7211b9021734016a749fe337396983c 2024-12-03T09:16:48,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6145b19e879146d889f22498a56573a4 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6145b19e879146d889f22498a56573a4 2024-12-03T09:16:48,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/76ed6a14fbb547c49d698f3390a05592 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/76ed6a14fbb547c49d698f3390a05592 2024-12-03T09:16:48,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/9aa8f893da484dc39635b0cb0edf3485 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/9aa8f893da484dc39635b0cb0edf3485 2024-12-03T09:16:48,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/663d6e1af3df472a9e2261512ed94c87 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/663d6e1af3df472a9e2261512ed94c87 2024-12-03T09:16:48,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/f3da1a78ba37476c936e05f0d549977a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/f3da1a78ba37476c936e05f0d549977a 2024-12-03T09:16:48,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6ba81c8484f74246ab6f62bdf8175347 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6ba81c8484f74246ab6f62bdf8175347 2024-12-03T09:16:48,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/e1215d9d93a44663b861d61b6cc8a344 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/e1215d9d93a44663b861d61b6cc8a344 2024-12-03T09:16:48,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/22c8d563137f4a388f6729880d93d18e to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/22c8d563137f4a388f6729880d93d18e 2024-12-03T09:16:48,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/3a1790c3ec364cec9ff6968831ace777 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/3a1790c3ec364cec9ff6968831ace777 2024-12-03T09:16:48,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/556be96e1fa04f28b80fbf41d7adfd5a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/556be96e1fa04f28b80fbf41d7adfd5a 2024-12-03T09:16:48,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6b778751b0c5439fabf825fb864ac84a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/6b778751b0c5439fabf825fb864ac84a 2024-12-03T09:16:48,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/fbb2370143744ab881a54d39c33ababa to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/fbb2370143744ab881a54d39c33ababa 2024-12-03T09:16:48,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/344312c578ef4d64b21383274720cf7a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/344312c578ef4d64b21383274720cf7a 2024-12-03T09:16:48,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/a8858734052749ce95f77c5d90b6e6ab to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/a8858734052749ce95f77c5d90b6e6ab 2024-12-03T09:16:48,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/5cdd74f59a4b4128bf89e31d79d42f38 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/5cdd74f59a4b4128bf89e31d79d42f38 2024-12-03T09:16:48,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/101f2cee823a47059c0b8fe59a8e9f11 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/101f2cee823a47059c0b8fe59a8e9f11 2024-12-03T09:16:48,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/9677350dd5104903a428bbf8cb453fa2 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/9677350dd5104903a428bbf8cb453fa2 2024-12-03T09:16:48,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/28eb360761644df0aee4ca5dac225e56 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/28eb360761644df0aee4ca5dac225e56 2024-12-03T09:16:48,328 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/recovered.edits/347.seqid, newMaxSeqId=347, maxSeqId=4 2024-12-03T09:16:48,328 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. 2024-12-03T09:16:48,328 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 750235cb928bf724493a7da0f57c7eb3: Waiting for close lock at 1733217406569Running coprocessor pre-close hooks at 1733217406569Disabling compacts and flushes for region at 1733217406569Disabling writes for close at 1733217406569Obtaining lock to block concurrent updates at 1733217406570 (+1 ms)Preparing flush snapshotting stores in 750235cb928bf724493a7da0f57c7eb3 at 1733217406570Finished memstore snapshotting TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733217406570Flushing stores of TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3. at 1733217406571 (+1 ms)Flushing 750235cb928bf724493a7da0f57c7eb3/A: creating writer at 1733217406571Flushing mob file 750235cb928bf724493a7da0f57c7eb3/A: appending metadata at 1733217406580 (+9 ms)Flushing mob file 750235cb928bf724493a7da0f57c7eb3/A: closing flushed file at 1733217406580Flushing 750235cb928bf724493a7da0f57c7eb3/A: appending metadata at 1733217406991 (+411 ms)Flushing 750235cb928bf724493a7da0f57c7eb3/A: closing flushed file at 1733217406991Flushing 750235cb928bf724493a7da0f57c7eb3/B: creating writer at 1733217407407 (+416 ms)Flushing 750235cb928bf724493a7da0f57c7eb3/B: appending metadata at 1733217407412 (+5 ms)Flushing 750235cb928bf724493a7da0f57c7eb3/B: closing flushed file at 1733217407412Flushing 750235cb928bf724493a7da0f57c7eb3/C: creating writer at 1733217407826 (+414 ms)Flushing 750235cb928bf724493a7da0f57c7eb3/C: appending metadata at 1733217407829 (+3 ms)Flushing 750235cb928bf724493a7da0f57c7eb3/C: closing flushed file at 1733217407829Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@485efc04: reopening flushed file at 1733217408242 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58413dc6: reopening flushed file at 1733217408250 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75923ed4: reopening flushed file at 1733217408254 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 750235cb928bf724493a7da0f57c7eb3 in 1690ms, sequenceid=344, compaction requested=true at 1733217408259 (+5 ms)Writing region close event to WAL at 1733217408326 (+67 ms)Running coprocessor post-close hooks at 1733217408328 (+2 ms)Closed at 1733217408328 2024-12-03T09:16:48,330 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,330 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=750235cb928bf724493a7da0f57c7eb3, regionState=CLOSED 2024-12-03T09:16:48,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 because future has completed 2024-12-03T09:16:48,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-03T09:16:48,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 750235cb928bf724493a7da0f57c7eb3, server=699c9baf3d29,42271,1733217224446 in 1.9160 sec 2024-12-03T09:16:48,334 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-12-03T09:16:48,334 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=750235cb928bf724493a7da0f57c7eb3, UNASSIGN in 1.9200 sec 2024-12-03T09:16:48,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-03T09:16:48,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9240 sec 2024-12-03T09:16:48,337 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217408336"}]},"ts":"1733217408336"} 2024-12-03T09:16:48,338 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T09:16:48,338 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T09:16:48,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9350 sec 2024-12-03T09:16:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T09:16:48,550 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:48,551 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-03T09:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:48,554 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T09:16:48,555 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:48,558 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,563 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C, FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/recovered.edits] 2024-12-03T09:16:48,568 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/1ba6b30343534713aff700b39b8b354f to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/1ba6b30343534713aff700b39b8b354f 2024-12-03T09:16:48,571 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/dd58ee07bfc44ac289eda68b43451378 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/dd58ee07bfc44ac289eda68b43451378 2024-12-03T09:16:48,573 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f8092073fa06493996334cedba91dd35 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/A/f8092073fa06493996334cedba91dd35 2024-12-03T09:16:48,578 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/09e9945ae5424a82afee2430a23da597 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/09e9945ae5424a82afee2430a23da597 2024-12-03T09:16:48,580 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/4234920d42274081a56798d1c06ac8c1 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/4234920d42274081a56798d1c06ac8c1 2024-12-03T09:16:48,582 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/699372d083f04b92b2994bca559142fb to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/B/699372d083f04b92b2994bca559142fb 2024-12-03T09:16:48,586 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/1029afe72195407da55cf6e5e29c00db to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/1029afe72195407da55cf6e5e29c00db 2024-12-03T09:16:48,588 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/7ce8bed9b3124b14a58bd1cbbef8f7dc to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/7ce8bed9b3124b14a58bd1cbbef8f7dc 2024-12-03T09:16:48,589 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/ba8458a098a34825addb753bca41f39a to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/C/ba8458a098a34825addb753bca41f39a 2024-12-03T09:16:48,593 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/recovered.edits/347.seqid to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3/recovered.edits/347.seqid 2024-12-03T09:16:48,594 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/default/TestAcidGuarantees/750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,595 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T09:16:48,595 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T09:16:48,596 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T09:16:48,599 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030f3a98f2f82b4d0197229d479de718a0_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030f3a98f2f82b4d0197229d479de718a0_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,601 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120315fd638e4c0d484b8d4e7389c9ac7af4_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120315fd638e4c0d484b8d4e7389c9ac7af4_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,602 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120316c2470fc5344709b3d51196d4c40690_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120316c2470fc5344709b3d51196d4c40690_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,603 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032667a0e5b7f941c2b27fc47ed1cfe983_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032667a0e5b7f941c2b27fc47ed1cfe983_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,604 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032cc9e7b7d5034a2f95c558fd2b10a578_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032cc9e7b7d5034a2f95c558fd2b10a578_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,606 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120330e1dc83eb3b4873a4dee8462b6045cf_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120330e1dc83eb3b4873a4dee8462b6045cf_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,607 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120346fc2e301b3d4c6bbf44a6eaa7a7d745_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120346fc2e301b3d4c6bbf44a6eaa7a7d745_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,608 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034f418804a3934cf897d04f2b45b3ca3a_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034f418804a3934cf897d04f2b45b3ca3a_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,609 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036025f3e696f24f00bf992f5650efa16a_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036025f3e696f24f00bf992f5650efa16a_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,611 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036ac1aed9390c49a0882c6416d12910c0_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036ac1aed9390c49a0882c6416d12910c0_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,612 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120372f6411eebe6442a8e26ad5ee3db7a36_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120372f6411eebe6442a8e26ad5ee3db7a36_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,614 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038534a2bcb0ac40cda696f288f0d393f3_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038534a2bcb0ac40cda696f288f0d393f3_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,615 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039d405f5c378544bdb5087790a4428300_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039d405f5c378544bdb5087790a4428300_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,617 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a30e147c48ff48b29d8ff2a2556f830b_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a30e147c48ff48b29d8ff2a2556f830b_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,618 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203abb1fb4e42984b589eedb476ff52e04e_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203abb1fb4e42984b589eedb476ff52e04e_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,619 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203adb879242cec4914b802c43c4d97b78d_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203adb879242cec4914b802c43c4d97b78d_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,621 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dbed9b524f8f4a8d837704bba97e5fd5_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203dbed9b524f8f4a8d837704bba97e5fd5_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,622 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e6f1423ab4634caa82668831377ff01f_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e6f1423ab4634caa82668831377ff01f_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,623 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e87f6568c52d491691f0ed4c0ee0d04f_750235cb928bf724493a7da0f57c7eb3 to hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e87f6568c52d491691f0ed4c0ee0d04f_750235cb928bf724493a7da0f57c7eb3 2024-12-03T09:16:48,624 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T09:16:48,627 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:48,630 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T09:16:48,632 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T09:16:48,633 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:48,633 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T09:16:48,633 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733217408633"}]},"ts":"9223372036854775807"} 2024-12-03T09:16:48,635 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T09:16:48,635 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 750235cb928bf724493a7da0f57c7eb3, NAME => 'TestAcidGuarantees,,1733217380994.750235cb928bf724493a7da0f57c7eb3.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T09:16:48,635 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T09:16:48,635 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733217408635"}]},"ts":"9223372036854775807"} 2024-12-03T09:16:48,636 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T09:16:48,637 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T09:16:48,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 86 msec 2024-12-03T09:16:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35781 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T09:16:48,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T09:16:48,660 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T09:16:48,671 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 250), OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=459 (was 560), ProcessCount=11 (was 11), AvailableMemoryMB=12763 (was 12775) 2024-12-03T09:16:48,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T09:16:48,671 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T09:16:48,671 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:16:48,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:48,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:48,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:16:48,672 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T09:16:48,672 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=855483586, stopped=false 2024-12-03T09:16:48,672 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=699c9baf3d29,35781,1733217223715 2024-12-03T09:16:48,736 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T09:16:48,736 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:16:48,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T09:16:48,736 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T09:16:48,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:16:48,737 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T09:16:48,737 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:16:48,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:48,738 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T09:16:48,738 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T09:16:48,739 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '699c9baf3d29,42271,1733217224446' ***** 2024-12-03T09:16:48,739 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T09:16:48,740 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T09:16:48,740 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T09:16:48,740 INFO [RS:0;699c9baf3d29:42271 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T09:16:48,740 INFO [RS:0;699c9baf3d29:42271 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T09:16:48,740 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(959): stopping server 699c9baf3d29,42271,1733217224446 2024-12-03T09:16:48,740 INFO [RS:0;699c9baf3d29:42271 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T09:16:48,741 INFO [RS:0;699c9baf3d29:42271 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;699c9baf3d29:42271. 2024-12-03T09:16:48,741 DEBUG [RS:0;699c9baf3d29:42271 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:16:48,741 DEBUG [RS:0;699c9baf3d29:42271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:48,741 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T09:16:48,741 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T09:16:48,741 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T09:16:48,741 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T09:16:48,742 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T09:16:48,742 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-03T09:16:48,742 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T09:16:48,742 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T09:16:48,742 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T09:16:48,742 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T09:16:48,742 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T09:16:48,742 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T09:16:48,742 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-03T09:16:48,763 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/.tmp/info/a4b6c75ee0f44d1d8470ddf9df7c7de7 is 95, key is TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12./info:/1733217256505/DeleteFamily/seqid=0 2024-12-03T09:16:48,765 INFO [regionserver/699c9baf3d29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T09:16:48,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742436_1612 (size=6514) 2024-12-03T09:16:48,942 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T09:16:49,000 INFO [regionserver/699c9baf3d29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T09:16:49,000 INFO [regionserver/699c9baf3d29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T09:16:49,143 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T09:16:49,168 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/.tmp/info/a4b6c75ee0f44d1d8470ddf9df7c7de7 2024-12-03T09:16:49,192 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/.tmp/ns/0c9b49aea3e943c78199a83352d7eedf is 93, key is TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12./ns:/1733217256469/DeleteFamily/seqid=0 2024-12-03T09:16:49,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742437_1613 (size=5906) 2024-12-03T09:16:49,343 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T09:16:49,543 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T09:16:49,597 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/.tmp/ns/0c9b49aea3e943c78199a83352d7eedf 2024-12-03T09:16:49,620 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/.tmp/rep_barrier/fa791a61a77d44d2a05a355e55fb48ac is 102, key is TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12./rep_barrier:/1733217256469/DeleteFamily/seqid=0 2024-12-03T09:16:49,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742438_1614 (size=6025) 2024-12-03T09:16:49,743 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T09:16:49,944 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T09:16:50,025 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/.tmp/rep_barrier/fa791a61a77d44d2a05a355e55fb48ac 2024-12-03T09:16:50,049 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/.tmp/table/4adcf9f71e794b9094769966dbb60e76 is 96, key is TestAcidGuarantees,,1733217228137.c49f86fc80f2af2d2b0cf03332e8cc12./table:/1733217256469/DeleteFamily/seqid=0 2024-12-03T09:16:50,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742439_1615 (size=5936) 2024-12-03T09:16:50,144 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T09:16:50,344 DEBUG [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T09:16:50,454 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/.tmp/table/4adcf9f71e794b9094769966dbb60e76 2024-12-03T09:16:50,465 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/.tmp/info/a4b6c75ee0f44d1d8470ddf9df7c7de7 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/info/a4b6c75ee0f44d1d8470ddf9df7c7de7 2024-12-03T09:16:50,469 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/info/a4b6c75ee0f44d1d8470ddf9df7c7de7, entries=12, sequenceid=90, filesize=6.4 K 2024-12-03T09:16:50,470 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/.tmp/ns/0c9b49aea3e943c78199a83352d7eedf as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/ns/0c9b49aea3e943c78199a83352d7eedf 2024-12-03T09:16:50,473 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/ns/0c9b49aea3e943c78199a83352d7eedf, entries=8, sequenceid=90, filesize=5.8 K 2024-12-03T09:16:50,474 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/.tmp/rep_barrier/fa791a61a77d44d2a05a355e55fb48ac as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/rep_barrier/fa791a61a77d44d2a05a355e55fb48ac 2024-12-03T09:16:50,477 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/rep_barrier/fa791a61a77d44d2a05a355e55fb48ac, entries=6, sequenceid=90, filesize=5.9 K 2024-12-03T09:16:50,478 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/.tmp/table/4adcf9f71e794b9094769966dbb60e76 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/table/4adcf9f71e794b9094769966dbb60e76 2024-12-03T09:16:50,481 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/table/4adcf9f71e794b9094769966dbb60e76, entries=7, sequenceid=90, filesize=5.8 K 2024-12-03T09:16:50,482 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false 2024-12-03T09:16:50,486 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-03T09:16:50,487 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T09:16:50,487 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T09:16:50,487 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733217408742Running coprocessor pre-close hooks at 1733217408742Disabling compacts and flushes for region at 1733217408742Disabling writes for close at 1733217408742Obtaining lock to block concurrent updates at 1733217408742Preparing flush snapshotting stores in 1588230740 at 1733217408742Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733217408743 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733217408744 (+1 ms)Flushing 1588230740/info: creating writer at 1733217408744Flushing 1588230740/info: appending metadata at 1733217408763 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733217408763Flushing 1588230740/ns: creating writer at 1733217409177 (+414 ms)Flushing 1588230740/ns: appending metadata at 1733217409192 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733217409192Flushing 1588230740/rep_barrier: creating writer at 1733217409605 (+413 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733217409620 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733217409620Flushing 1588230740/table: creating writer at 1733217410033 (+413 ms)Flushing 1588230740/table: appending metadata at 1733217410049 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733217410049Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79859c6c: reopening flushed file at 1733217410464 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@322566ac: reopening flushed file at 1733217410470 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62138fa1: reopening flushed file at 1733217410473 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fdf0c7e: reopening flushed file at 1733217410477 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false at 1733217410482 (+5 ms)Writing region close event to WAL at 1733217410484 (+2 ms)Running coprocessor post-close hooks at 1733217410487 (+3 ms)Closed at 1733217410487 2024-12-03T09:16:50,488 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T09:16:50,545 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(976): stopping server 699c9baf3d29,42271,1733217224446; all regions closed. 2024-12-03T09:16:50,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741834_1010 (size=25344) 2024-12-03T09:16:50,554 DEBUG [RS:0;699c9baf3d29:42271 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/oldWALs 2024-12-03T09:16:50,554 INFO [RS:0;699c9baf3d29:42271 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 699c9baf3d29%2C42271%2C1733217224446.meta:.meta(num 1733217227774) 2024-12-03T09:16:50,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741832_1008 (size=15673370) 2024-12-03T09:16:50,559 DEBUG [RS:0;699c9baf3d29:42271 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/oldWALs 2024-12-03T09:16:50,559 INFO [RS:0;699c9baf3d29:42271 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 699c9baf3d29%2C42271%2C1733217224446:(num 1733217227155) 2024-12-03T09:16:50,559 DEBUG [RS:0;699c9baf3d29:42271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:16:50,559 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T09:16:50,559 INFO [RS:0;699c9baf3d29:42271 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T09:16:50,560 INFO [RS:0;699c9baf3d29:42271 {}] hbase.ChoreService(370): Chore service for: regionserver/699c9baf3d29:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T09:16:50,560 INFO [RS:0;699c9baf3d29:42271 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T09:16:50,560 INFO [regionserver/699c9baf3d29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T09:16:50,561 INFO [RS:0;699c9baf3d29:42271 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42271 2024-12-03T09:16:50,630 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/699c9baf3d29,42271,1733217224446 2024-12-03T09:16:50,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T09:16:50,630 INFO [RS:0;699c9baf3d29:42271 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T09:16:50,641 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [699c9baf3d29,42271,1733217224446] 2024-12-03T09:16:50,651 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/699c9baf3d29,42271,1733217224446 already deleted, retry=false 2024-12-03T09:16:50,651 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 699c9baf3d29,42271,1733217224446 expired; onlineServers=0 2024-12-03T09:16:50,652 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '699c9baf3d29,35781,1733217223715' ***** 2024-12-03T09:16:50,652 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T09:16:50,652 INFO [M:0;699c9baf3d29:35781 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T09:16:50,652 INFO [M:0;699c9baf3d29:35781 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T09:16:50,652 DEBUG [M:0;699c9baf3d29:35781 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T09:16:50,652 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T09:16:50,652 DEBUG [M:0;699c9baf3d29:35781 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T09:16:50,652 DEBUG [master/699c9baf3d29:0:becomeActiveMaster-HFileCleaner.large.0-1733217226850 {}] cleaner.HFileCleaner(306): Exit Thread[master/699c9baf3d29:0:becomeActiveMaster-HFileCleaner.large.0-1733217226850,5,FailOnTimeoutGroup] 2024-12-03T09:16:50,652 DEBUG [master/699c9baf3d29:0:becomeActiveMaster-HFileCleaner.small.0-1733217226851 {}] cleaner.HFileCleaner(306): Exit Thread[master/699c9baf3d29:0:becomeActiveMaster-HFileCleaner.small.0-1733217226851,5,FailOnTimeoutGroup] 2024-12-03T09:16:50,652 INFO [M:0;699c9baf3d29:35781 {}] hbase.ChoreService(370): Chore service for: master/699c9baf3d29:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T09:16:50,652 INFO [M:0;699c9baf3d29:35781 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T09:16:50,653 DEBUG [M:0;699c9baf3d29:35781 {}] master.HMaster(1795): Stopping service threads 2024-12-03T09:16:50,653 INFO [M:0;699c9baf3d29:35781 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T09:16:50,653 INFO [M:0;699c9baf3d29:35781 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T09:16:50,653 ERROR [M:0;699c9baf3d29:35781 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-03T09:16:50,654 INFO [M:0;699c9baf3d29:35781 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T09:16:50,654 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T09:16:50,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T09:16:50,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:16:50,662 DEBUG [M:0;699c9baf3d29:35781 {}] zookeeper.ZKUtil(347): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T09:16:50,662 WARN [M:0;699c9baf3d29:35781 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T09:16:50,662 INFO [M:0;699c9baf3d29:35781 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/.lastflushedseqids 2024-12-03T09:16:50,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742440_1616 (size=72) 2024-12-03T09:16:50,742 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T09:16:50,742 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42271-0x1019a77fcb50001, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T09:16:50,742 INFO [RS:0;699c9baf3d29:42271 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T09:16:50,743 INFO [RS:0;699c9baf3d29:42271 {}] regionserver.HRegionServer(1031): Exiting; stopping=699c9baf3d29,42271,1733217224446; zookeeper connection closed. 2024-12-03T09:16:50,743 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5439e9a4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5439e9a4 2024-12-03T09:16:50,744 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T09:16:51,073 INFO [M:0;699c9baf3d29:35781 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T09:16:51,073 INFO [M:0;699c9baf3d29:35781 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T09:16:51,073 DEBUG [M:0;699c9baf3d29:35781 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T09:16:51,073 INFO [M:0;699c9baf3d29:35781 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:16:51,073 DEBUG [M:0;699c9baf3d29:35781 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:16:51,074 DEBUG [M:0;699c9baf3d29:35781 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T09:16:51,074 DEBUG [M:0;699c9baf3d29:35781 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:16:51,074 INFO [M:0;699c9baf3d29:35781 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=769.26 KB heapSize=942.16 KB 2024-12-03T09:16:51,096 DEBUG [M:0;699c9baf3d29:35781 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e72000ee587049e6a84ed0fd038c3ae3 is 82, key is hbase:meta,,1/info:regioninfo/1733217227847/Put/seqid=0 2024-12-03T09:16:51,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742441_1617 (size=5672) 2024-12-03T09:16:51,501 INFO [M:0;699c9baf3d29:35781 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2138 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e72000ee587049e6a84ed0fd038c3ae3 2024-12-03T09:16:51,534 DEBUG [M:0;699c9baf3d29:35781 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21b040e6524541018889bf302ba268a5 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8F/proc:d/1733217384082/Put/seqid=0 2024-12-03T09:16:51,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742442_1618 (size=44646) 2024-12-03T09:16:51,938 INFO [M:0;699c9baf3d29:35781 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=768.71 KB at sequenceid=2138 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21b040e6524541018889bf302ba268a5 2024-12-03T09:16:51,944 INFO [M:0;699c9baf3d29:35781 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 21b040e6524541018889bf302ba268a5 2024-12-03T09:16:51,963 DEBUG [M:0;699c9baf3d29:35781 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3e7c1a12940540e38ac8ef91dffa6810 is 69, key is 699c9baf3d29,42271,1733217224446/rs:state/1733217226923/Put/seqid=0 2024-12-03T09:16:51,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073742443_1619 (size=5156) 2024-12-03T09:16:52,368 INFO [M:0;699c9baf3d29:35781 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2138 (bloomFilter=true), to=hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3e7c1a12940540e38ac8ef91dffa6810 2024-12-03T09:16:52,379 DEBUG [M:0;699c9baf3d29:35781 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e72000ee587049e6a84ed0fd038c3ae3 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e72000ee587049e6a84ed0fd038c3ae3 2024-12-03T09:16:52,383 INFO [M:0;699c9baf3d29:35781 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e72000ee587049e6a84ed0fd038c3ae3, entries=8, sequenceid=2138, filesize=5.5 K 2024-12-03T09:16:52,384 DEBUG [M:0;699c9baf3d29:35781 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21b040e6524541018889bf302ba268a5 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/21b040e6524541018889bf302ba268a5 2024-12-03T09:16:52,388 INFO [M:0;699c9baf3d29:35781 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 21b040e6524541018889bf302ba268a5 2024-12-03T09:16:52,388 INFO [M:0;699c9baf3d29:35781 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/21b040e6524541018889bf302ba268a5, entries=166, sequenceid=2138, filesize=43.6 K 2024-12-03T09:16:52,389 DEBUG [M:0;699c9baf3d29:35781 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3e7c1a12940540e38ac8ef91dffa6810 as hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3e7c1a12940540e38ac8ef91dffa6810 2024-12-03T09:16:52,392 INFO [M:0;699c9baf3d29:35781 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43771/user/jenkins/test-data/b8951da8-374f-e057-a3fe-0f1b9dc1cb09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3e7c1a12940540e38ac8ef91dffa6810, entries=1, sequenceid=2138, filesize=5.0 K 2024-12-03T09:16:52,392 INFO [M:0;699c9baf3d29:35781 {}] regionserver.HRegion(3140): Finished flush of dataSize ~769.26 KB/787724, heapSize ~941.87 KB/964472, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1318ms, sequenceid=2138, compaction requested=false 2024-12-03T09:16:52,395 INFO [M:0;699c9baf3d29:35781 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:16:52,395 DEBUG [M:0;699c9baf3d29:35781 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733217411073Disabling compacts and flushes for region at 1733217411073Disabling writes for close at 1733217411074 (+1 ms)Obtaining lock to block concurrent updates at 1733217411074Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733217411074Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=787724, getHeapSize=964712, getOffHeapSize=0, getCellsCount=2329 at 1733217411074Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733217411075 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733217411075Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733217411096 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733217411096Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733217411509 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733217411533 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733217411533Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733217411945 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733217411963 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733217411963Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b454809: reopening flushed file at 1733217412378 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@629dd910: reopening flushed file at 1733217412384 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6be95573: reopening flushed file at 1733217412388 (+4 ms)Finished flush of dataSize ~769.26 KB/787724, heapSize ~941.87 KB/964472, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1318ms, sequenceid=2138, compaction requested=false at 1733217412392 (+4 ms)Writing region close event to WAL at 1733217412395 (+3 ms)Closed at 1733217412395 2024-12-03T09:16:52,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39265 is added to blk_1073741830_1006 (size=927877) 2024-12-03T09:16:52,398 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T09:16:52,398 INFO [M:0;699c9baf3d29:35781 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T09:16:52,398 INFO [M:0;699c9baf3d29:35781 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35781 2024-12-03T09:16:52,398 INFO [M:0;699c9baf3d29:35781 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T09:16:52,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T09:16:52,551 INFO [M:0;699c9baf3d29:35781 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T09:16:52,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35781-0x1019a77fcb50000, quorum=127.0.0.1:58590, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T09:16:52,561 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@53343f2d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T09:16:52,565 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3ff8ab72{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T09:16:52,565 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T09:16:52,566 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a247cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T09:16:52,566 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@849d63d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/hadoop.log.dir/,STOPPED} 2024-12-03T09:16:52,569 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T09:16:52,570 WARN [BP-777680306-172.17.0.2-1733217220182 heartbeating to localhost/127.0.0.1:43771 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T09:16:52,570 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T09:16:52,570 WARN [BP-777680306-172.17.0.2-1733217220182 heartbeating to localhost/127.0.0.1:43771 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-777680306-172.17.0.2-1733217220182 (Datanode Uuid df211a26-18bf-4a7e-a99b-44cda7c05900) service to localhost/127.0.0.1:43771 2024-12-03T09:16:52,571 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/cluster_265ee787-60f6-c6d8-cc9e-dd886dec81cb/data/data1/current/BP-777680306-172.17.0.2-1733217220182 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T09:16:52,572 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/cluster_265ee787-60f6-c6d8-cc9e-dd886dec81cb/data/data2/current/BP-777680306-172.17.0.2-1733217220182 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T09:16:52,572 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T09:16:52,579 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5f062aca{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T09:16:52,580 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4fee885d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T09:16:52,580 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T09:16:52,580 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@707ebc91{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T09:16:52,580 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b490999{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/65b19e0c-e173-4619-dcdd-a8725a3bc0b3/hadoop.log.dir/,STOPPED} 2024-12-03T09:16:52,594 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T09:16:52,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down